From 245b4d3b7915c661c81a41e786441e1108c4fe2d Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:17:50 -0600 Subject: [PATCH] Preserve cancelled workspaces after controllerless Stop (#18820) Admit cancelled workspace preservation after an audited Stop acknowledgement when no live controller received an interrupt. Retain the strict dispatched receipt for provider termination and operator handoff. Existing source, lease, coordinator and result ownership checks remain enforced. Validated with complete Stop/preservation suites (134 tests), real-file recovery, workspace typecheck and build, and green hosted CI. Greptile and Paperclip both reviewed the exact head at 5/5 with no unresolved findings. Co-Authored-By: Paperclip --- doc/execution-semantics.md | 5 ++ .../services/acknowledged-native-stop.test.ts | 4 +- .../src/services/acknowledged-native-stop.ts | 10 ++-- ...e-cancelled-workspace-preservation.test.ts | 47 +++++++++++++++++-- .../native-failed-workspace-recovery.ts | 8 ++-- .../src/services/workspace-recovery.test.ts | 35 ++++++++++++++ 6 files changed, 96 insertions(+), 13 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 9e190669123..96e3ca64ffb 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2281,6 +2281,11 @@ not grant board Stop or continuation authority, change the cancelled result, or prove physical termination. Normal recovery stops and retains the same allocation, saves its files, and obtains a fresh stop receipt before cleanup can apply the original destroy disposition. It starts no provider turn. +An audited acknowledgement with `dispatched: false` also admits this preservation +when Stop found no live controller. The worker retains the exact prepared source +and obtains independent provider-stop evidence through normal recovery. It does +not convert the no-op acknowledgement into a dispatched interrupt receipt or +permission to continue the cancelled turn. A reusable lease can also owe native copyback: the export-repair action must select its exact immutable descriptor and the finalizer's accepted successful result under the same company, task, run and completion contract. Lease reuse diff --git a/server/src/services/acknowledged-native-stop.test.ts b/server/src/services/acknowledged-native-stop.test.ts index f8772e5761e..860e8a25c4d 100644 --- a/server/src/services/acknowledged-native-stop.test.ts +++ b/server/src/services/acknowledged-native-stop.test.ts @@ -10,7 +10,7 @@ const run = { id: "run", companyId: "company", nativeIssueId: "issue", status: " it("recognizes the native receipt used by the Stop button", () => { expect(isAcknowledgedNativeStop(run)).toBe(true); }); -it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" }, +it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" }, { dispatched: false }, { scope: "subtree" }, { acknowledgementAuditId: undefined }])("refuses unrelated or incomplete receipts %j", change => { expect(isAcknowledgedNativeStop({ ...run, resultJson: { ...run.resultJson, nativeCancellation: { ...run.resultJson.nativeCancellation, ...change } } })).toBe(false); @@ -23,7 +23,7 @@ it("recognizes an audited handoff without granting operator Stop semantics", () expect(isAcknowledgedNativeReassignmentStop({ ...handoff, status: "running" })).toBe(false); expect(isAcknowledgedNativeReassignmentStop(run)).toBe(false); }); -it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" }, +it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" }, { dispatched: false }, { scope: "subtree" }, { acknowledgementAuditId: undefined }])("requires an audited handoff receipt %j", change => { expect(isAcknowledgedNativeReassignmentStop({ ...run, resultJson: { reassignmentStopRequested: true, nativeCancellation: { ...run.resultJson.nativeCancellation, ...change } } })).toBe(false); diff --git a/server/src/services/acknowledged-native-stop.ts b/server/src/services/acknowledged-native-stop.ts index 09124602f0d..e140b6968ad 100644 --- a/server/src/services/acknowledged-native-stop.ts +++ b/server/src/services/acknowledged-native-stop.ts @@ -8,8 +8,7 @@ import { validatePrpEvent } from "../vendor/paperclip-runner/index.js"; import { nativeSha256 } from "./native-runtime/canonical.js"; import { isConfirmedCursorCloudTerminalObservation } from "./native-runtime/cursor-cloud-runtime.js"; -/** A server-recorded run-only Stop must not manufacture a recovery incident. */ -export function hasAcknowledgedNativeRunStopReceipt(run: { +export function hasAcknowledgedNativeRunStopRequest(run: { id: string; companyId: string; status: string; nativeIssueId: string | null; resultJson: Record | null; }): boolean { @@ -18,11 +17,16 @@ export function hasAcknowledgedNativeRunStopReceipt(run: { return intent?.schema === "paperclip.native-cancellation.v1" && intent.runId === run.id && intent.companyId === run.companyId && intent.issueId === run.nativeIssueId && intent.scope === "run" && intent.reasonCode === "cancellation_run_only" && - intent.dispatchState === "acknowledged" && intent.dispatched === true && + intent.dispatchState === "acknowledged" && typeof intent.dispatched === "boolean" && typeof intent.intentAuditId === "string" && Boolean(intent.intentAuditId) && typeof intent.acknowledgementAuditId === "string" && Boolean(intent.acknowledgementAuditId); } +export function hasAcknowledgedNativeRunStopReceipt(run: Parameters[0]): boolean { + return hasAcknowledgedNativeRunStopRequest(run) && + (run.resultJson?.nativeCancellation as Record).dispatched === true; +} + export function hasAcknowledgedNativeStopIntent(run: Parameters[0]): boolean { return run.resultJson?.cancelledByActorType === "user" && typeof run.resultJson.cancelledByUserId === "string" && Boolean(run.resultJson.cancelledByUserId) && diff --git a/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts b/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts index 72a8df28337..0916bd8c90f 100644 --- a/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts +++ b/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts @@ -5,7 +5,7 @@ import { agents, agentWakeupRequests, companies, completionContracts, createDb, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db"; import { recoveryPostgresSupport, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; import { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; -import { hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js"; +import { hasAcknowledgedNativeRunStopReceipt, hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js"; import { requestWorkspaceRecovery } from "../workspace-recovery.js"; import { inspectWorkspaceRecoverySource } from "../workspace-recovery-source.js"; import { preserveCancelledNativeWorkspaces, preserveNativeFailedWorkspace, @@ -120,14 +120,53 @@ import { preserveCancelledNativeWorkspaces, preserveNativeFailedWorkspace, } finally { await worker.stop(); } }, 120_000); - it.each(["pending", "undispatched", "foreign-company", "foreign-issue", "foreign-run", "turn-scope", + it.each(["native-sync", "remote-workspace", "startup-scan"] as const)("preserves files when Stop finds no live controller (%s)", async kind => { + const f = await seed(); + const nativeCancellation = { ...f.nativeCancellation, dispatched: false }; + const resultJson = { ...f.run.resultJson, nativeCancellation, cancelledByActorType: "user", cancelledByUserId: "operator" }; + const runnerProfileJson = kind === "native-sync" ? { nativeWorkspaceSync: { + schema: "paperclip.native-workspace-sync/v3", state: "prepared", workspaceId: f.workspaceId, + leaseId: f.leaseId, providerLeaseId: f.reference.providerLeaseId, remoteCwd: f.reference.remoteCwd, + baselineSha256: "a".repeat(64), descriptorSha256: "b".repeat(64), finalHostSha256: null, + resourceDisposition: "destroy", + }, nativeResourceDisposition: "destroy" } : f.run.runnerProfileJson; + await db.update(heartbeatRuns).set({ resultJson, runnerProfileJson }).where(eq(heartbeatRuns.id, f.runId)); + if (kind === "native-sync") await db.update(executionWorkspaces).set({ metadata: {} }) + .where(and(eq(executionWorkspaces.companyId, f.companyId), eq(executionWorkspaces.id, f.workspaceId))); + const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + try { + if (kind === "startup-scan") await preserveCancelledNativeWorkspaces(db); + else await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready; + const [retained] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(retained.resultJson).toMatchObject({ ...resultJson, + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.leaseId] } }); + expect(retained).toMatchObject({ status: "cancelled", finishedAt: f.run.finishedAt, + runnerProfileJson, costAccountingPending: true, usageJson: f.run.usageJson }); + expect(hasAcknowledgedNativeRunStopReceipt(retained)).toBe(false); + expect(hasAcknowledgedNativeStopIntent(retained)).toBe(false); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + expect(lease).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure", + metadata: { sandboxStopAndRetain: { runId: f.runId, leaseId: f.leaseId } } }); + expect(lease.metadata).not.toHaveProperty("remoteExecutionTermination"); + const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId)); + expect((await inspectWorkspaceRecoverySource(db, { companyId: f.companyId, issueId: f.issueId, + runId: f.runId, actionId: action.id })).sources).toHaveLength(1); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("in_progress"); + expect(worker.nextWakeAt()).toBeNull(); + } finally { await worker.stop(); } + }, 120_000); + + it.each(["pending", "missing-dispatch", "invalid-dispatch", "foreign-company", "foreign-issue", "foreign-run", "turn-scope", "empty-intent-audit", "missing-ack-audit", "empty-ack-audit", "live-owner", "live-controller", "accepted-result", "foreign-allocation", "foreign-workspace", "foreign-cwd", "saved-reference", "unknown-reference", "running"] as const) ("does not admit preservation for %s", async invalid => { const f = await seed(); - const intent: Record = { ...f.nativeCancellation }; + const intent: Record = { ...f.nativeCancellation, dispatched: false }; if (invalid === "pending") intent.dispatchState = "pending"; - if (invalid === "undispatched") intent.dispatched = false; + if (invalid === "missing-dispatch") delete intent.dispatched; + if (invalid === "invalid-dispatch") intent.dispatched = "false"; if (invalid === "foreign-company") intent.companyId = randomUUID(); if (invalid === "foreign-issue") intent.issueId = randomUUID(); if (invalid === "foreign-run") intent.runId = randomUUID(); diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 0900ecc6aa0..9681695bef3 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -11,7 +11,7 @@ import { logActivity } from "../activity-log.js"; import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; import type { NativeRestartRecoveryClaim } from "./native-restart-recovery.js"; import { NativeWorkspaceFinalizationBusyError } from "./native-workspace-finalization-ownership.js"; -import { hasAcknowledgedNativeRunStopReceipt } from "../acknowledged-native-stop.js"; +import { hasAcknowledgedNativeRunStopRequest } from "../acknowledged-native-stop.js"; import { logger } from "../../middleware/logger.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { preserveRemoteWorkspacePreparationFailure } from "./native-workspace-finalization-state.js"; @@ -81,7 +81,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick Date.now())) return; const currentReference = workspaceRecoveryReference(run.runnerProfileJson); if (!currentReference || currentReference.state !== "prepared" @@ -195,7 +195,7 @@ export async function preserveCancelledNativeWorkspaces(db: Db) { for (const candidate of candidates) { cancelledWorkspaceCursors.set(db, candidate.runId); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, candidate.runId)); - if (run && hasAcknowledgedNativeRunStopReceipt(run)) { + if (run && hasAcknowledgedNativeRunStopRequest(run)) { try { await preserveNativeFailedWorkspace(db, run, { allowPreterminal: true, requireCancelledStop: true }); } catch (error) { if (error instanceof NativeWorkspaceFinalizationBusyError) continue; @@ -221,7 +221,7 @@ export function registerCancelledWorkspacePreservationWork(db: Db, type Entry = { id: string; companyId: string; dueAt: number }; type Candidate = Pick; - const eligible = (run: Candidate) => run.nativeIssueId !== null && hasAcknowledgedNativeRunStopReceipt(run) + const eligible = (run: Candidate) => run.nativeIssueId !== null && hasAcknowledgedNativeRunStopRequest(run) && workspaceRecoveryReference(run.runnerProfileJson)?.state === "prepared" && !hasRequiredWorkspaceRecovery(run.resultJson) && !nativeRunHasNoWorkspaceAccess(run); const scopeCondition = (scope?: CancelledWorkspacePreservationScope) => scope ? and( diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index dde9ded92b5..0e2efa8864b 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -31,6 +31,7 @@ import { workspaceOperationService } from "./workspace-operations.js"; import { instanceSettingsService } from "./instance-settings.js"; import { projectTaskInterruption, taskInterruptionPresentation } from "./task-interruption-presentation.js"; import * as nativeWorkspaceSync from "./native-runtime/native-workspace-sync.js"; +import { registerCancelledWorkspacePreservationWork } from "./native-runtime/native-failed-workspace-recovery.js"; import { GitShallowMetadataError } from "@paperclipai/adapter-utils/git-workspace-sync"; import { readSystemRecovery } from "./system-recovery-state.js"; import { adoptSystemRecovery, updateSystemRecovery } from "./system-recovery.js"; @@ -225,6 +226,40 @@ describe("automatic retained workspace recovery", () => { const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!; const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!; + it("recovers files after Stop finds no live controller without replaying the cancelled chat", async () => { + const f = await seed("native", true, "destroy", false, true), original = await run(f.runId); + await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId)); + await db.update(issues).set({ conversationAgentId: original.agentId, conversationUserId: "operator", + conversationState: "active", status: "in_review" }).where(eq(issues.id, f.issueId)); + const nativeCancellation = { schema: "paperclip.native-cancellation.v1", runId: f.runId, companyId: f.companyId, + issueId: f.issueId, scope: "run", reasonCode: "cancellation_run_only", dispatchState: "acknowledged", dispatched: false, + intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID() }; + await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "cancelled", resultJson: { + nativeCancellation, cancelledByActorType: "user", cancelledByUserId: "operator", + executionCancellation: { state: "requested", requestedAt: new Date().toISOString() }, + } }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(environmentLeases).set({ status: "active", releasedAt: null, cleanupStatus: null, + metadata: f.lease.metadata }).where(eq(environmentLeases.id, f.lease.id)); + const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + try { + await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready; + const [preservation] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId)); + expect(preservation).toBeDefined(); + await recoverRetainedWorkspace({ ...f.input, actionId: preservation.id }); + expect(readSystemRecovery((await action(preservation.id)).evidence)?.state).toBe("complete"); + expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n"); + expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n"); + expect(await run(f.runId)).toMatchObject({ status: "cancelled", resultJson: { nativeCancellation }, + runnerProfileJson: { nativeWorkspaceSync: { state: "finalized" } } }); + expect(f.runtime.retryPendingSandboxTeardown).toHaveBeenCalled(); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0].metadata) + .toHaveProperty("remoteExecutionTermination"); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("in_review"); + } finally { await worker.stop(); } + }, 30_000); + async function failedWarmSource(exportOnly = false, hops = 0) { const f = hops ? await warmHandoffSource(hops) : await seed("native", true, "keep_running"); if (hops) await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, f.runId));