Preserve cancelled workspaces after controllerless Stop (#18820)

Admit cancelled workspace preservation after an audited Stop acknowledgement when no live controller received an interrupt. Retain the strict dispatched receipt for provider termination and operator handoff. Existing source, lease, coordinator and result ownership checks remain enforced.

Validated with complete Stop/preservation suites (134 tests), real-file recovery, workspace typecheck and build, and green hosted CI. Greptile and Paperclip both reviewed the exact head at 5/5 with no unresolved findings.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/execution-semantics.md

@@ -2281,6 +2281,11 @@ not grant board Stop or continuation authority, change the cancelled result,
 or prove physical termination. Normal recovery stops and retains the same
 allocation, saves its files, and obtains a fresh stop receipt before cleanup
 can apply the original destroy disposition. It starts no provider turn.
+An audited acknowledgement with `dispatched: false` also admits this preservation
+when Stop found no live controller. The worker retains the exact prepared source
+and obtains independent provider-stop evidence through normal recovery. It does
+not convert the no-op acknowledgement into a dispatched interrupt receipt or
+permission to continue the cancelled turn.
 A reusable lease can also owe native copyback: the export-repair action must
 select its exact immutable descriptor and the finalizer's accepted successful
 result under the same company, task, run and completion contract. Lease reuse

server/src/services/acknowledged-native-stop.test.ts

@@ -10,7 +10,7 @@ const run = { id: "run", companyId: "company", nativeIssueId: "issue", status: "
 it("recognizes the native receipt used by the Stop button", () => {
   expect(isAcknowledgedNativeStop(run)).toBe(true);
 });
-it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" },
+it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" }, { dispatched: false },
   { scope: "subtree" }, { acknowledgementAuditId: undefined }])("refuses unrelated or incomplete receipts %j", change => {
   expect(isAcknowledgedNativeStop({ ...run, resultJson: { ...run.resultJson,
     nativeCancellation: { ...run.resultJson.nativeCancellation, ...change } } })).toBe(false);
@@ -23,7 +23,7 @@ it("recognizes an audited handoff without granting operator Stop semantics", ()
   expect(isAcknowledgedNativeReassignmentStop({ ...handoff, status: "running" })).toBe(false);
   expect(isAcknowledgedNativeReassignmentStop(run)).toBe(false);
 });
-it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" },
+it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" }, { dispatched: false },
   { scope: "subtree" }, { acknowledgementAuditId: undefined }])("requires an audited handoff receipt %j", change => {
   expect(isAcknowledgedNativeReassignmentStop({ ...run, resultJson: { reassignmentStopRequested: true,
     nativeCancellation: { ...run.resultJson.nativeCancellation, ...change } } })).toBe(false);

server/src/services/acknowledged-native-stop.ts

@@ -8,8 +8,7 @@ import { validatePrpEvent } from "../vendor/paperclip-runner/index.js";
 import { nativeSha256 } from "./native-runtime/canonical.js";
 import { isConfirmedCursorCloudTerminalObservation } from "./native-runtime/cursor-cloud-runtime.js";
 
-/** A server-recorded run-only Stop must not manufacture a recovery incident. */
-export function hasAcknowledgedNativeRunStopReceipt(run: {
+export function hasAcknowledgedNativeRunStopRequest(run: {
   id: string; companyId: string; status: string; nativeIssueId: string | null;
   resultJson: Record<string, unknown> | null;
 }): boolean {
@@ -18,11 +17,16 @@ export function hasAcknowledgedNativeRunStopReceipt(run: {
   return intent?.schema === "paperclip.native-cancellation.v1" && intent.runId === run.id &&
     intent.companyId === run.companyId && intent.issueId === run.nativeIssueId &&
     intent.scope === "run" && intent.reasonCode === "cancellation_run_only" &&
-    intent.dispatchState === "acknowledged" && intent.dispatched === true &&
+    intent.dispatchState === "acknowledged" && typeof intent.dispatched === "boolean" &&
     typeof intent.intentAuditId === "string" && Boolean(intent.intentAuditId) &&
     typeof intent.acknowledgementAuditId === "string" && Boolean(intent.acknowledgementAuditId);
 }
 
+export function hasAcknowledgedNativeRunStopReceipt(run: Parameters<typeof hasAcknowledgedNativeRunStopRequest>[0]): boolean {
+  return hasAcknowledgedNativeRunStopRequest(run) &&
+    (run.resultJson?.nativeCancellation as Record<string, unknown>).dispatched === true;
+}
+
 export function hasAcknowledgedNativeStopIntent(run: Parameters<typeof hasAcknowledgedNativeRunStopReceipt>[0]): boolean {
   return run.resultJson?.cancelledByActorType === "user" &&
     typeof run.resultJson.cancelledByUserId === "string" && Boolean(run.resultJson.cancelledByUserId) &&

server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts

@@ -5,7 +5,7 @@ import { agents, agentWakeupRequests, companies, completionContracts, createDb,
   executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db";
 import { recoveryPostgresSupport, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js";
 import { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
-import { hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js";
+import { hasAcknowledgedNativeRunStopReceipt, hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js";
 import { requestWorkspaceRecovery } from "../workspace-recovery.js";
 import { inspectWorkspaceRecoverySource } from "../workspace-recovery-source.js";
 import { preserveCancelledNativeWorkspaces, preserveNativeFailedWorkspace,
@@ -120,14 +120,53 @@ import { preserveCancelledNativeWorkspaces, preserveNativeFailedWorkspace,
     } finally { await worker.stop(); }
   }, 120_000);
 
-  it.each(["pending", "undispatched", "foreign-company", "foreign-issue", "foreign-run", "turn-scope",
+  it.each(["native-sync", "remote-workspace", "startup-scan"] as const)("preserves files when Stop finds no live controller (%s)", async kind => {
+    const f = await seed();
+    const nativeCancellation = { ...f.nativeCancellation, dispatched: false };
+    const resultJson = { ...f.run.resultJson, nativeCancellation, cancelledByActorType: "user", cancelledByUserId: "operator" };
+    const runnerProfileJson = kind === "native-sync" ? { nativeWorkspaceSync: {
+      schema: "paperclip.native-workspace-sync/v3", state: "prepared", workspaceId: f.workspaceId,
+      leaseId: f.leaseId, providerLeaseId: f.reference.providerLeaseId, remoteCwd: f.reference.remoteCwd,
+      baselineSha256: "a".repeat(64), descriptorSha256: "b".repeat(64), finalHostSha256: null,
+      resourceDisposition: "destroy",
+    }, nativeResourceDisposition: "destroy" } : f.run.runnerProfileJson;
+    await db.update(heartbeatRuns).set({ resultJson, runnerProfileJson }).where(eq(heartbeatRuns.id, f.runId));
+    if (kind === "native-sync") await db.update(executionWorkspaces).set({ metadata: {} })
+      .where(and(eq(executionWorkspaces.companyId, f.companyId), eq(executionWorkspaces.id, f.workspaceId)));
+    const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
+    try {
+      if (kind === "startup-scan") await preserveCancelledNativeWorkspaces(db);
+      else await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready;
+      const [retained] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
+      expect(retained.resultJson).toMatchObject({ ...resultJson,
+        workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.leaseId] } });
+      expect(retained).toMatchObject({ status: "cancelled", finishedAt: f.run.finishedAt,
+        runnerProfileJson, costAccountingPending: true, usageJson: f.run.usageJson });
+      expect(hasAcknowledgedNativeRunStopReceipt(retained)).toBe(false);
+      expect(hasAcknowledgedNativeStopIntent(retained)).toBe(false);
+      const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId));
+      expect(lease).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure",
+          metadata: { sandboxStopAndRetain: { runId: f.runId, leaseId: f.leaseId } } });
+      expect(lease.metadata).not.toHaveProperty("remoteExecutionTermination");
+      const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId));
+      expect((await inspectWorkspaceRecoverySource(db, { companyId: f.companyId, issueId: f.issueId,
+        runId: f.runId, actionId: action.id })).sources).toHaveLength(1);
+      expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
+      expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0);
+      expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("in_progress");
+      expect(worker.nextWakeAt()).toBeNull();
+    } finally { await worker.stop(); }
+  }, 120_000);
+
+  it.each(["pending", "missing-dispatch", "invalid-dispatch", "foreign-company", "foreign-issue", "foreign-run", "turn-scope",
     "empty-intent-audit", "missing-ack-audit", "empty-ack-audit", "live-owner", "live-controller", "accepted-result",
     "foreign-allocation", "foreign-workspace", "foreign-cwd", "saved-reference", "unknown-reference", "running"] as const)
   ("does not admit preservation for %s", async invalid => {
     const f = await seed();
-    const intent: Record<string, unknown> = { ...f.nativeCancellation };
+    const intent: Record<string, unknown> = { ...f.nativeCancellation, dispatched: false };
     if (invalid === "pending") intent.dispatchState = "pending";
-    if (invalid === "undispatched") intent.dispatched = false;
+    if (invalid === "missing-dispatch") delete intent.dispatched;
+    if (invalid === "invalid-dispatch") intent.dispatched = "false";
     if (invalid === "foreign-company") intent.companyId = randomUUID();
     if (invalid === "foreign-issue") intent.issueId = randomUUID();
     if (invalid === "foreign-run") intent.runId = randomUUID();

server/src/services/native-runtime/native-failed-workspace-recovery.ts

@@ -11,7 +11,7 @@ import { logActivity } from "../activity-log.js";
 import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js";
 import type { NativeRestartRecoveryClaim } from "./native-restart-recovery.js";
 import { NativeWorkspaceFinalizationBusyError } from "./native-workspace-finalization-ownership.js";
-import { hasAcknowledgedNativeRunStopReceipt } from "../acknowledged-native-stop.js";
+import { hasAcknowledgedNativeRunStopRequest } from "../acknowledged-native-stop.js";
 import { logger } from "../../middleware/logger.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
 import { preserveRemoteWorkspacePreparationFailure } from "./native-workspace-finalization-state.js";
@@ -81,7 +81,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick<typeo
       eq(heartbeatRuns.id, selected.id), eq(heartbeatRuns.companyId, selected.companyId),
     )).for("update");
     if (!run || run.runtimeMode !== "native" || (!options.allowPreterminal && run.status !== "failed")) return;
-    if (options.requireCancelledStop && (run.status !== "cancelled" || !hasAcknowledgedNativeRunStopReceipt(run)
+    if (options.requireCancelledStop && (run.status !== "cancelled" || !hasAcknowledgedNativeRunStopRequest(run)
       || run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt.getTime() > Date.now())) return;
     const currentReference = workspaceRecoveryReference(run.runnerProfileJson);
     if (!currentReference || currentReference.state !== "prepared"
@@ -195,7 +195,7 @@ export async function preserveCancelledNativeWorkspaces(db: Db) {
   for (const candidate of candidates) {
     cancelledWorkspaceCursors.set(db, candidate.runId);
     const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, candidate.runId));
-    if (run && hasAcknowledgedNativeRunStopReceipt(run)) {
+    if (run && hasAcknowledgedNativeRunStopRequest(run)) {
       try { await preserveNativeFailedWorkspace(db, run, { allowPreterminal: true, requireCancelledStop: true }); }
       catch (error) {
         if (error instanceof NativeWorkspaceFinalizationBusyError) continue;
@@ -221,7 +221,7 @@ export function registerCancelledWorkspacePreservationWork(db: Db,
   type Entry = { id: string; companyId: string; dueAt: number };
   type Candidate = Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId" | "agentId" | "issueId" | "nativeIssueId"
     | "runtimeMode" | "driverKind" | "status" | "resultJson" | "runnerProfileJson">;
-  const eligible = (run: Candidate) => run.nativeIssueId !== null && hasAcknowledgedNativeRunStopReceipt(run)
+  const eligible = (run: Candidate) => run.nativeIssueId !== null && hasAcknowledgedNativeRunStopRequest(run)
     && workspaceRecoveryReference(run.runnerProfileJson)?.state === "prepared"
     && !hasRequiredWorkspaceRecovery(run.resultJson) && !nativeRunHasNoWorkspaceAccess(run);
   const scopeCondition = (scope?: CancelledWorkspacePreservationScope) => scope ? and(

server/src/services/workspace-recovery.test.ts

@@ -31,6 +31,7 @@ import { workspaceOperationService } from "./workspace-operations.js";
 import { instanceSettingsService } from "./instance-settings.js";
 import { projectTaskInterruption, taskInterruptionPresentation } from "./task-interruption-presentation.js";
 import * as nativeWorkspaceSync from "./native-runtime/native-workspace-sync.js";
+import { registerCancelledWorkspacePreservationWork } from "./native-runtime/native-failed-workspace-recovery.js";
 import { GitShallowMetadataError } from "@paperclipai/adapter-utils/git-workspace-sync";
 import { readSystemRecovery } from "./system-recovery-state.js";
 import { adoptSystemRecovery, updateSystemRecovery } from "./system-recovery.js";
@@ -225,6 +226,40 @@ describe("automatic retained workspace recovery", () => {
   const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!;
   const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!;
 
+  it("recovers files after Stop finds no live controller without replaying the cancelled chat", async () => {
+    const f = await seed("native", true, "destroy", false, true), original = await run(f.runId);
+    await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId));
+    await db.update(issues).set({ conversationAgentId: original.agentId, conversationUserId: "operator",
+      conversationState: "active", status: "in_review" }).where(eq(issues.id, f.issueId));
+    const nativeCancellation = { schema: "paperclip.native-cancellation.v1", runId: f.runId, companyId: f.companyId,
+      issueId: f.issueId, scope: "run", reasonCode: "cancellation_run_only", dispatchState: "acknowledged", dispatched: false,
+      intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID() };
+    await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "cancelled", resultJson: {
+      nativeCancellation, cancelledByActorType: "user", cancelledByUserId: "operator",
+      executionCancellation: { state: "requested", requestedAt: new Date().toISOString() },
+    } }).where(eq(heartbeatRuns.id, f.runId));
+    await db.update(environmentLeases).set({ status: "active", releasedAt: null, cleanupStatus: null,
+      metadata: f.lease.metadata }).where(eq(environmentLeases.id, f.lease.id));
+    const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
+    try {
+      await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready;
+      const [preservation] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId));
+      expect(preservation).toBeDefined();
+      await recoverRetainedWorkspace({ ...f.input, actionId: preservation.id });
+      expect(readSystemRecovery((await action(preservation.id)).evidence)?.state).toBe("complete");
+      expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n");
+      expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n");
+      expect(await run(f.runId)).toMatchObject({ status: "cancelled", resultJson: { nativeCancellation },
+        runnerProfileJson: { nativeWorkspaceSync: { state: "finalized" } } });
+      expect(f.runtime.retryPendingSandboxTeardown).toHaveBeenCalled();
+      expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0].metadata)
+        .toHaveProperty("remoteExecutionTermination");
+      expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
+      expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0);
+      expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("in_review");
+    } finally { await worker.stop(); }
+  }, 30_000);
+
   async function failedWarmSource(exportOnly = false, hops = 0) {
     const f = hops ? await warmHandoffSource(hops) : await seed("native", true, "keep_running");
     if (hops) await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, f.runId));