diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 06def5c40dc..daf4bba6ad9 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -535,10 +535,13 @@ in its stopped sandbox. When the run used a reusable (warm) sandbox, the stop leaves that lease `released` under its reusable policy with a stop-and-retain receipt. Cancelled-workspace preservation admits that stopped source, as it does an active or ephemeral one. It records the save obligation for the exact writer -run while the workspace is still bound to that writer. Paperclip's workspace -recovery then saves those files, which releases the hold. It never replays the -stopped run. While that save is in progress, its last progress counts as the -ended writer's activity for the holder cutoff. A waiter keeps waiting instead of +run while the workspace is still bound to that writer. A Stop during an +accepted result's copy-back retries ends that copy-back, and no finalization +exports those files afterwards. Preservation admits that writer as well and +leaves the accepted result uncommitted. Other accepted results keep their own +export repair path. Paperclip's workspace recovery then saves those files, which +releases the hold. It never replays the stopped run. While that save is in +progress, its last progress counts as the ended writer's activity for the holder cutoff. A waiter keeps waiting instead of failing setup. The save's retry budget bounds that wait. A save that exhausts its retries is handed to the board as a named recovery action. After that hand-off, the cutoff no longer advances. diff --git a/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts b/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts index 2acdb9912dc..23b1847db48 100644 --- a/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts +++ b/server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts @@ -75,6 +75,19 @@ import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0); } + async function acceptResult(f: Awaited>) { + const [contract] = await db.insert(completionContracts).values({ companyId: f.companyId, issueId: f.issueId, + revision: 1, schemaVersion: "v1", policyVersion: "v1", risk: "low", completionAuthority: "agent", + incompleteCriteriaPolicy: "block", contractJson: {}, canonicalSha256: "a".repeat(64), + createdByActorType: "system", createdByActorId: "test" }).returning(); + await db.update(heartbeatRuns).set({ completionContractId: contract.id }).where(eq(heartbeatRuns.id, f.runId)); + const [result] = await db.insert(nativeRunResults).values({ companyId: f.companyId, issueId: f.issueId, runId: f.runId, + completionContractId: contract.id, serverFingerprint: "b".repeat(64), schemaStatus: "accepted", + resultJson: {}, canonicalSha256: "c".repeat(64) }).returning(); + await db.update(nativeRunFinalizations).set({ resultId: result.id }).where(eq(nativeRunFinalizations.runId, f.runId)); + return result; + } + it("reconstructs prepared remote files after an audited lifecycle Stop without a board actor", async () => { const f = await seed(); const [historical] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, @@ -263,17 +276,7 @@ import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; }).where(eq(heartbeatRuns.id, f.runId)); if (invalid === "live-owner") await db.update(nativeRunFinalizations).set({ leaseOwner: randomUUID(), leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId)); - if (invalid === "accepted-result") { - const [contract] = await db.insert(completionContracts).values({ companyId: f.companyId, issueId: f.issueId, - revision: 1, schemaVersion: "v1", policyVersion: "v1", risk: "low", completionAuthority: "agent", - incompleteCriteriaPolicy: "block", contractJson: {}, canonicalSha256: "a".repeat(64), - createdByActorType: "system", createdByActorId: "test" }).returning(); - await db.update(heartbeatRuns).set({ completionContractId: contract.id }).where(eq(heartbeatRuns.id, f.runId)); - const [result] = await db.insert(nativeRunResults).values({ companyId: f.companyId, issueId: f.issueId, runId: f.runId, - completionContractId: contract.id, serverFingerprint: "b".repeat(64), schemaStatus: "accepted", - resultJson: {}, canonicalSha256: "c".repeat(64) }).returning(); - await db.update(nativeRunFinalizations).set({ resultId: result.id }).where(eq(nativeRunFinalizations.runId, f.runId)); - } + if (invalid === "accepted-result") await acceptResult(f); const errors: unknown[] = []; const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { errors.push(error); } }); try { @@ -345,6 +348,52 @@ import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; } finally { await worker.stop(); } }, 120_000); + // Stop during an accepted result's copy-back retries ends the copy-back. No + // finalization exports those files afterwards, so preservation keeps them. + it.each(["worker", "startup-scan", "stopped-sandbox"] as const)("preserves files of an accepted result whose copy-back was stopped (%s)", async kind => { + const f = await seed(); + const result = await acceptResult(f); + const nativeCancellation = { ...f.nativeCancellation, dispatched: false }; + await db.update(heartbeatRuns).set({ nativePhase: "retryable_failure", error: "Cancelled by a board operator", + resultJson: { nativeCancellation, cancelledByActorType: "user", cancelledByUserId: "operator" } }) + .where(eq(heartbeatRuns.id, f.runId)); + await db.update(nativeRunFinalizations).set({ failureCode: "native_retry_cancelled", + failureDetail: { originalFailureCode: "native_workspace_sync_out_failed", workspaceFinalizeAttempt: 1 } }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + if (kind === "stopped-sandbox") { + // The Stop's resource release already stopped and kept the sandbox. + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + const stopping = (await prepareSandboxStopAndRetain(db, lease))!; + expect(await settleStopOnlyCleanup(db, stopping, { attemptId: String(stopping.metadata!.pendingCleanupAttemptId), + receipt: { providerLeaseId: f.reference.providerLeaseId, state: "stopped" } })) + .toMatchObject({ status: "released", leasePolicy: "ephemeral", cleanupStatus: "success" }); + } + const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + try { + if (kind === "startup-scan") await preserveCancelledNativeWorkspaces(db); + else await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready; + const [retained] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(retained.resultJson).toMatchObject({ nativeCancellation, workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.leaseId] } }); + expect(retained).toMatchObject({ status: "cancelled", finishedAt: f.run.finishedAt }); + const actions = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId)); + expect(actions).toHaveLength(1); + expect(actions[0].evidence).toMatchObject({ runId: f.runId, workspaceRestoreRecovery: { leaseIds: [f.leaseId] } }); + expect((await inspectWorkspaceRecoverySource(db, { companyId: f.companyId, issueId: f.issueId, + runId: f.runId, actionId: actions[0].id })).sources).toHaveLength(1); + // The accepted result stays recorded and is never committed or replayed. + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]) + .toMatchObject({ phase: "terminal_failure", resultId: result.id, failureCode: "native_retry_cancelled", leaseOwner: null }); + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.workspaceId)); + expect(await readRemoteWorkspaceWriterHold(db, { companyId: f.companyId, + authority: readRemoteWorkspaceAuthority(workspace.metadata)! })).toMatchObject({ kind: "held", writer: { runId: f.runId, ended: true } }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("in_progress"); + if (kind !== "startup-scan") expect(worker.nextWakeAt()).toBeNull(); + } finally { await worker.stop(); } + }, 120_000); + it("fences recovery of a stopped reusable source behind later allocation owners and live writers", async () => { const f = await seed(); await stopAndRetainReusableSource(f); diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 38b06bacafd..9ec7ac330fe 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -68,6 +68,14 @@ export async function preserveTaskRemoteWorkspaceWriter(db: Db, companyId: strin catch (error) { if (!(error instanceof NativeWorkspaceFinalizationBusyError)) throw error; } } +/** A Stop during an accepted result's copy-back retries ends that copy-back. + * No finalization exports those files afterwards, so they are kept like the + * files of any other stopped run. Other accepted results keep their own export + * repair path. */ +function stoppedAcceptedCopyback(coordinator: Pick) { + return coordinator.phase === "terminal_failure" && coordinator.failureCode === "native_retry_cancelled"; +} + export async function preserveNativeFailedWorkspace(db: Db, selected: Pick, options: { allowPreterminal?: boolean; requireCancelledStop?: boolean; restartRecovery?: NativeRestartRecoveryClaim } = {}) { @@ -94,7 +102,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick'nativeWorkspaceSync'->>'state' = 'prepared' OR ${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'state' = 'prepared')`, sql`${heartbeatRuns.resultJson}->'nativeCancellation'->>'dispatchState' = 'acknowledged'`, cursor ? gt(heartbeatRuns.id, cursor) : undefined, @@ -289,7 +298,7 @@ export function registerCancelledWorkspacePreservationWork(db: Db, eq(nativeRunFinalizations.runId, run.id), eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.issueId, run.nativeIssueId!), )); - if (finalization?.resultId) return null; + if (finalization?.resultId && !stoppedAcceptedCopyback(finalization)) return null; if (finalization?.phase !== "terminal_failure" || finalization.leaseOwner) return { ...entry, dueAt: Date.now() + 30_000 }; if (await cancelledWorkspaceSource(db, run)) await preserveNativeFailedWorkspace(db, run, { allowPreterminal: true, requireCancelledStop: true });