diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 9e190669123..96e3ca64ffb 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2281,6 +2281,11 @@ not grant board Stop or continuation authority, change the cancelled result, or prove physical termination. Normal recovery stops and retains the same allocation, saves its files, and obtains a fresh stop receipt before cleanup can apply the original destroy disposition. It starts no provider turn. +An audited acknowledgement with `dispatched: false` also admits this preservation +when Stop found no live controller. The worker retains the exact prepared source +and obtains independent provider-stop evidence through normal recovery. It does +not convert the no-op acknowledgement into a dispatched interrupt receipt or +permission to continue the cancelled turn. A reusable lease can also owe native copyback: the export-repair action must select its exact immutable descriptor and the finalizer's accepted successful result under the same company, task, run and completion contract. Lease reuse diff --git a/server/src/services/acknowledged-native-stop.test.ts b/server/src/services/acknowledged-native-stop.test.ts index f8772e5761e..860e8a25c4d 100644 --- a/server/src/services/acknowledged-native-stop.test.ts +++ b/server/src/services/acknowledged-native-stop.test.ts @@ -10,7 +10,7 @@ const run = { id: "run", companyId: "company", nativeIssueId: "issue", status: " it("recognizes the native receipt used by the Stop button", () => { expect(isAcknowledgedNativeStop(run)).toBe(true); }); -it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" }, +it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" }, { dispatched: false }, { scope: "subtree" }, { acknowledgementAuditId: undefined }])("refuses unrelated or incomplete receipts %j", change => { expect(isAcknowledgedNativeStop({ ...run, resultJson: { ...run.resultJson, nativeCancellation: { ...run.resultJson.nativeCancellation, ...change } } })).toBe(false); @@ -23,7 +23,7 @@ it("recognizes an audited handoff without granting operator Stop semantics", () expect(isAcknowledgedNativeReassignmentStop({ ...handoff, status: "running" })).toBe(false); expect(isAcknowledgedNativeReassignmentStop(run)).toBe(false); }); -it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" }, +it.each([{ runId: "other" }, { companyId: "other" }, { issueId: "other" }, { dispatchState: "pending" }, { dispatched: false }, { scope: "subtree" }, { acknowledgementAuditId: undefined }])("requires an audited handoff receipt %j", change => { expect(isAcknowledgedNativeReassignmentStop({ ...run, resultJson: { reassignmentStopRequested: true, nativeCancellation: { ...run.resultJson.nativeCancellation, ...change } } })).toBe(false); diff --git a/server/src/services/acknowledged-native-stop.ts b/server/src/services/acknowledged-native-stop.ts index 09124602f0d..e140b6968ad 100644 --- a/server/src/services/acknowledged-native-stop.ts +++ b/server/src/services/acknowledged-native-stop.ts @@ -8,8 +8,7 @@ import { validatePrpEvent } from "../vendor/paperclip-runner/index.js"; import { nativeSha256 } from "./native-runtime/canonical.js"; import { isConfirmedCursorCloudTerminalObservation } from "./native-runtime/cursor-cloud-runtime.js"; -/** A server-recorded run-only Stop must not manufacture a recovery incident. */ -export function hasAcknowledgedNativeRunStopReceipt(run: { +export function hasAcknowledgedNativeRunStopRequest(run: { id: string; companyId: string; status: string; nativeIssueId: string | null; resultJson: Record | null; }): boolean { @@ -18,11 +17,16 @@ export function hasAcknowledgedNativeRunStopReceipt(run: { return intent?.schema === "paperclip.native-cancellation.v1" && intent.runId === run.id && intent.companyId === run.companyId && intent.issueId === run.nativeIssueId && intent.scope === "run" && intent.reasonCode === "cancellation_run_only" && - intent.dispatchState === "acknowledged" && intent.dispatched === true && + intent.dispatchState === "acknowledged" && typeof intent.dispatched === "boolean" && typeof intent.intentAuditId === "string" && Boolean(intent.intentAuditId) && typeof intent.acknowledgementAuditId === "string" && Boolean(intent.acknowledgementAuditId); } +export function hasAcknowledgedNativeRunStopReceipt(run: Parameters[0]): boolean { + return hasAcknowledgedNativeRunStopRequest(run) && + (run.resultJson?.nativeCancellation as Record).dispatched === true; +} + export function hasAcknowledgedNativeStopIntent(run: Parameters[0]): boolean { return run.resultJson?.cancelledByActorType === "user" && typeof run.resultJson.cancelledByUserId === "string" && Boolean(run.resultJson.cancelledByUserId) && diff --git a/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts b/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts index 72a8df28337..0916bd8c90f 100644 --- a/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts +++ b/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts @@ -5,7 +5,7 @@ import { agents, agentWakeupRequests, companies, completionContracts, createDb, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db"; import { recoveryPostgresSupport, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; import { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; -import { hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js"; +import { hasAcknowledgedNativeRunStopReceipt, hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js"; import { requestWorkspaceRecovery } from "../workspace-recovery.js"; import { inspectWorkspaceRecoverySource } from "../workspace-recovery-source.js"; import { preserveCancelledNativeWorkspaces, preserveNativeFailedWorkspace, @@ -120,14 +120,53 @@ import { preserveCancelledNativeWorkspaces, preserveNativeFailedWorkspace, } finally { await worker.stop(); } }, 120_000); - it.each(["pending", "undispatched", "foreign-company", "foreign-issue", "foreign-run", "turn-scope", + it.each(["native-sync", "remote-workspace", "startup-scan"] as const)("preserves files when Stop finds no live controller (%s)", async kind => { + const f = await seed(); + const nativeCancellation = { ...f.nativeCancellation, dispatched: false }; + const resultJson = { ...f.run.resultJson, nativeCancellation, cancelledByActorType: "user", cancelledByUserId: "operator" }; + const runnerProfileJson = kind === "native-sync" ? { nativeWorkspaceSync: { + schema: "paperclip.native-workspace-sync/v3", state: "prepared", workspaceId: f.workspaceId, + leaseId: f.leaseId, providerLeaseId: f.reference.providerLeaseId, remoteCwd: f.reference.remoteCwd, + baselineSha256: "a".repeat(64), descriptorSha256: "b".repeat(64), finalHostSha256: null, + resourceDisposition: "destroy", + }, nativeResourceDisposition: "destroy" } : f.run.runnerProfileJson; + await db.update(heartbeatRuns).set({ resultJson, runnerProfileJson }).where(eq(heartbeatRuns.id, f.runId)); + if (kind === "native-sync") await db.update(executionWorkspaces).set({ metadata: {} }) + .where(and(eq(executionWorkspaces.companyId, f.companyId), eq(executionWorkspaces.id, f.workspaceId))); + const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + try { + if (kind === "startup-scan") await preserveCancelledNativeWorkspaces(db); + else await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready; + const [retained] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(retained.resultJson).toMatchObject({ ...resultJson, + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.leaseId] } }); + expect(retained).toMatchObject({ status: "cancelled", finishedAt: f.run.finishedAt, + runnerProfileJson, costAccountingPending: true, usageJson: f.run.usageJson }); + expect(hasAcknowledgedNativeRunStopReceipt(retained)).toBe(false); + expect(hasAcknowledgedNativeStopIntent(retained)).toBe(false); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + expect(lease).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure", + metadata: { sandboxStopAndRetain: { runId: f.runId, leaseId: f.leaseId } } }); + expect(lease.metadata).not.toHaveProperty("remoteExecutionTermination"); + const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId)); + expect((await inspectWorkspaceRecoverySource(db, { companyId: f.companyId, issueId: f.issueId, + runId: f.runId, actionId: action.id })).sources).toHaveLength(1); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("in_progress"); + expect(worker.nextWakeAt()).toBeNull(); + } finally { await worker.stop(); } + }, 120_000); + + it.each(["pending", "missing-dispatch", "invalid-dispatch", "foreign-company", "foreign-issue", "foreign-run", "turn-scope", "empty-intent-audit", "missing-ack-audit", "empty-ack-audit", "live-owner", "live-controller", "accepted-result", "foreign-allocation", "foreign-workspace", "foreign-cwd", "saved-reference", "unknown-reference", "running"] as const) ("does not admit preservation for %s", async invalid => { const f = await seed(); - const intent: Record = { ...f.nativeCancellation }; + const intent: Record = { ...f.nativeCancellation, dispatched: false }; if (invalid === "pending") intent.dispatchState = "pending"; - if (invalid === "undispatched") intent.dispatched = false; + if (invalid === "missing-dispatch") delete intent.dispatched; + if (invalid === "invalid-dispatch") intent.dispatched = "false"; if (invalid === "foreign-company") intent.companyId = randomUUID(); if (invalid === "foreign-issue") intent.issueId = randomUUID(); if (invalid === "foreign-run") intent.runId = randomUUID(); diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 0900ecc6aa0..9681695bef3 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -11,7 +11,7 @@ import { logActivity } from "../activity-log.js"; import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; import type { NativeRestartRecoveryClaim } from "./native-restart-recovery.js"; import { NativeWorkspaceFinalizationBusyError } from "./native-workspace-finalization-ownership.js"; -import { hasAcknowledgedNativeRunStopReceipt } from "../acknowledged-native-stop.js"; +import { hasAcknowledgedNativeRunStopRequest } from "../acknowledged-native-stop.js"; import { logger } from "../../middleware/logger.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { preserveRemoteWorkspacePreparationFailure } from "./native-workspace-finalization-state.js"; @@ -81,7 +81,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick Date.now())) return; const currentReference = workspaceRecoveryReference(run.runnerProfileJson); if (!currentReference || currentReference.state !== "prepared" @@ -195,7 +195,7 @@ export async function preserveCancelledNativeWorkspaces(db: Db) { for (const candidate of candidates) { cancelledWorkspaceCursors.set(db, candidate.runId); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, candidate.runId)); - if (run && hasAcknowledgedNativeRunStopReceipt(run)) { + if (run && hasAcknowledgedNativeRunStopRequest(run)) { try { await preserveNativeFailedWorkspace(db, run, { allowPreterminal: true, requireCancelledStop: true }); } catch (error) { if (error instanceof NativeWorkspaceFinalizationBusyError) continue; @@ -221,7 +221,7 @@ export function registerCancelledWorkspacePreservationWork(db: Db, type Entry = { id: string; companyId: string; dueAt: number }; type Candidate = Pick; - const eligible = (run: Candidate) => run.nativeIssueId !== null && hasAcknowledgedNativeRunStopReceipt(run) + const eligible = (run: Candidate) => run.nativeIssueId !== null && hasAcknowledgedNativeRunStopRequest(run) && workspaceRecoveryReference(run.runnerProfileJson)?.state === "prepared" && !hasRequiredWorkspaceRecovery(run.resultJson) && !nativeRunHasNoWorkspaceAccess(run); const scopeCondition = (scope?: CancelledWorkspacePreservationScope) => scope ? and( diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index dde9ded92b5..0e2efa8864b 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -31,6 +31,7 @@ import { workspaceOperationService } from "./workspace-operations.js"; import { instanceSettingsService } from "./instance-settings.js"; import { projectTaskInterruption, taskInterruptionPresentation } from "./task-interruption-presentation.js"; import * as nativeWorkspaceSync from "./native-runtime/native-workspace-sync.js"; +import { registerCancelledWorkspacePreservationWork } from "./native-runtime/native-failed-workspace-recovery.js"; import { GitShallowMetadataError } from "@paperclipai/adapter-utils/git-workspace-sync"; import { readSystemRecovery } from "./system-recovery-state.js"; import { adoptSystemRecovery, updateSystemRecovery } from "./system-recovery.js"; @@ -225,6 +226,40 @@ describe("automatic retained workspace recovery", () => { const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!; const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!; + it("recovers files after Stop finds no live controller without replaying the cancelled chat", async () => { + const f = await seed("native", true, "destroy", false, true), original = await run(f.runId); + await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId)); + await db.update(issues).set({ conversationAgentId: original.agentId, conversationUserId: "operator", + conversationState: "active", status: "in_review" }).where(eq(issues.id, f.issueId)); + const nativeCancellation = { schema: "paperclip.native-cancellation.v1", runId: f.runId, companyId: f.companyId, + issueId: f.issueId, scope: "run", reasonCode: "cancellation_run_only", dispatchState: "acknowledged", dispatched: false, + intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID() }; + await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "cancelled", resultJson: { + nativeCancellation, cancelledByActorType: "user", cancelledByUserId: "operator", + executionCancellation: { state: "requested", requestedAt: new Date().toISOString() }, + } }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(environmentLeases).set({ status: "active", releasedAt: null, cleanupStatus: null, + metadata: f.lease.metadata }).where(eq(environmentLeases.id, f.lease.id)); + const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + try { + await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready; + const [preservation] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId)); + expect(preservation).toBeDefined(); + await recoverRetainedWorkspace({ ...f.input, actionId: preservation.id }); + expect(readSystemRecovery((await action(preservation.id)).evidence)?.state).toBe("complete"); + expect(await readFile(path.join(f.host, "saved.txt"), "utf8")).toBe("finished work\n"); + expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n"); + expect(await run(f.runId)).toMatchObject({ status: "cancelled", resultJson: { nativeCancellation }, + runnerProfileJson: { nativeWorkspaceSync: { state: "finalized" } } }); + expect(f.runtime.retryPendingSandboxTeardown).toHaveBeenCalled(); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0].metadata) + .toHaveProperty("remoteExecutionTermination"); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("in_review"); + } finally { await worker.stop(); } + }, 30_000); + async function failedWarmSource(exportOnly = false, hops = 0) { const f = hops ? await warmHandoffSource(hops) : await seed("native", true, "keep_running"); if (hops) await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, f.runId));