diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 2c5937f9dc2..69abce42052 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3679,7 +3679,7 @@ another provider attempt, so restarts the run already survived cannot block it. A disposition request that also ends without a result consumes the marker and fails closed as before. -Provider execution and control-plane finalization have different clocks. A healthy provider can think or execute a long tool without output. Once execution settles, recovery and finalization control steps have a 60-second deadline, checked on startup and every 15 seconds. With a healthy database and scheduler, an abandoned transition must be repaired or surfaced within 90 seconds. Terminal persistence must not wait on provider cleanup or publication; a late finalizer cannot change a reassigned or closed task or release another run's locks. In particular, a done report from a run whose task now belongs to another agent, a user or nobody is kept as an accepted result but records `completion_owner_superseded` and preserves the task status for the current owner. A finalizer retired because a newer run already committed the task's decision (`native_finalization_superseded`) also ends its run, so the run never holds the task or the runs queued behind it; the result is kept for reference and never applied, and the task's decision does not change. The native run projection worker ends a run that an older release left running in that state, at startup and whenever it is signalled. Historical ambiguous runs are never automatically replayed after an upgrade. +Provider execution and control-plane finalization have different clocks. A healthy provider can think or execute a long tool without output. Once execution settles, recovery and finalization control steps have a 60-second deadline, checked on startup and every 15 seconds. With a healthy database and scheduler, an abandoned transition must be repaired or surfaced within 90 seconds. Terminal persistence must not wait on provider cleanup or publication; a late finalizer cannot change a reassigned or closed task or release another run's locks. In particular, a done report from a run whose task now belongs to another agent, a user or nobody is kept as an accepted result but records `completion_owner_superseded` and preserves the task status for the current owner. A finalizer retired because a newer run already committed the task's decision (`native_finalization_superseded`) also ends its run, so the run never holds the task or the runs queued behind it; the result is kept for reference and never applied, and the task's decision does not change. The native run projection worker ends a run that an older release left running in that state, at startup and whenever it is signalled. When that retirement interrupted the run's workspace export, the run's files are still unsaved and the run still holds the task's workspace as its writer, so the same worker records them as a retained-file hold (the stopped sandbox is kept). The system workspace recovery then saves them from that sandbox within its bounded retry budget and frees the workspace. If every attempt fails, the task continues from its last committed save: only that run's workspace epoch is set aside (its location stays on the workspace record), the readable files are attached to the task unapplied, and the task shows a plain notice. The superseded result is never applied in either case. Historical ambiguous runs are never automatically replayed after an upgrade. A control deadline bounds the settlement of one attempt, never a later attempt of the same run. While a native run's finalization owner holds a live lease (for example, a controller that reclaimed the run after a server restart) or has a same-run resume scheduled, that owner is responsible for the run. The reconciler clears a run deadline left by an earlier attempt instead of failing the resumed work. When the live owner's own settlement deadline is due, the reconciler waits until that lease lapses before it surfaces the deadline. A run whose deadline genuinely expires fails with `execution_finalization_deadline_exceeded` and shows plain wording with Retry: its saved work is kept, and actions with unconfirmed outcomes are not repeated. Runs saved by earlier releases with the previous operator-facing wording show the same plain wording when the task is read; no migration is needed. diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 7d2de469019..bc37846ba7f 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -19,6 +19,7 @@ import { pinFailedObservedWorkspace } from "./native-observed-workspace-preserva import { hasConfirmedSandboxStopAndRetain } from "../sandbox-stop-and-retain.js"; import { preparedRemoteWorkspaceSource } from "../remote-workspace-lifecycle.js"; import { releaseUnchangedRemoteWorkspaceWriterById } from "../remote-workspaces.js"; +import { isSupersededWorkspaceExport, markSupersededWorkspaceExportHold } from "./native-superseded-workspace-export.js"; const reusableLeasePolicies = ["reuse_by_environment", "reuse_by_execution_workspace"]; @@ -103,8 +104,12 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick Promise = endSupersededNativeFinalizationRuns) { + endSuperseded: (db: Db, runIds: readonly string[]) => Promise = endSupersededNativeFinalizationRuns, + preserveExport: (db: Db, runId: string) => Promise<{ recheckAt: number | null }> = + (db, runId) => preserveSupersededWorkspaceExport(db, runId, preserveNativeFailedWorkspace)) { function candidates(scope?: NativeRunProjectionScope, after?: string) { return db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, issueId: nativeRunFinalizations.issueId }) .from(nativeRunFinalizations) @@ -34,7 +38,8 @@ export function nativeRunProjectionWork(db: Db, project = projectCommittedRun, .innerJoin(issues, and(eq(issues.id, nativeRunFinalizations.issueId), eq(issues.companyId, nativeRunFinalizations.companyId))) .where(and(or(and(eq(nativeRunFinalizations.phase, "committed"), pendingNativeRunProjectionCondition(db), - not(pendingNativeStatusEffectsCondition(db))), supersededRunStillRunningCondition()), + not(pendingNativeStatusEffectsCondition(db))), supersededRunStillRunningCondition(), + supersededWorkspaceExportUnpreservedCondition()), scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined, scope?.issueId ? eq(nativeRunFinalizations.issueId, scope.issueId) : undefined, scope?.runId ? eq(heartbeatRuns.id, scope.runId) : undefined, @@ -66,6 +71,18 @@ export function nativeRunProjectionWork(db: Db, project = projectCommittedRun, await endSuperseded(db, [entry.id]); return (await candidates(scope)).length ? { ...entry, dueAt: Date.now() + 30_000 } : null; } + const [unsavedExport] = await db.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations) + .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, nativeRunFinalizations.runId), + eq(heartbeatRuns.companyId, nativeRunFinalizations.companyId))) + .where(and(eq(nativeRunFinalizations.runId, entry.id), eq(nativeRunFinalizations.companyId, entry.companyId), + supersededWorkspaceExportUnpreservedCondition())); + if (unsavedExport) { + // An ended superseded run still holds its workspace with unsaved files. + // Record the files for automatic recovery; a refusal rests until the + // next start or signal, except while its sandbox stop is still arriving. + const { recheckAt } = await preserveExport(db, entry.id); + return recheckAt === null ? null : { ...entry, dueAt: recheckAt }; + } const [run] = await db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, status: heartbeatRuns.status, finishedAt: heartbeatRuns.finishedAt, nativePhase: heartbeatRuns.nativePhase, runtimeMode: heartbeatRuns.runtimeMode, nativeIssueId: heartbeatRuns.nativeIssueId, diff --git a/server/src/services/native-runtime/native-superseded-workspace-export.ts b/server/src/services/native-runtime/native-superseded-workspace-export.ts new file mode 100644 index 00000000000..30f3da2c689 --- /dev/null +++ b/server/src/services/native-runtime/native-superseded-workspace-export.ts @@ -0,0 +1,85 @@ +import { and, eq, inArray, isNotNull, isNull, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared"; +import { SUPERSEDED_WORKSPACE_EXPORT_KEY, SUPERSEDED_WORKSPACE_EXPORT_SCHEMA } from "../workspace-recovery-fallback-discovery.js"; +import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js"; +import { hasRequiredWorkspaceRecovery } from "../workspace-restore-recovery-state.js"; + +/** The finalizer's verdict for a result a newer run already decided (see native-run-finalizer). */ +const SUPERSEDED = "native_finalization_superseded"; +/** How long a just-ended run's source may wait for its sandbox stop receipt before the check rests. */ +const STOP_RECEIPT_WAIT_MS = 60 * 60_000; +const STOP_RECEIPT_RECHECK_MS = 60_000; + +type Coordinator = Pick; + +/** + * A finalizer retired as superseded while it was exporting the run's + * workspace. The result is never applied, but the run's files were not saved + * either, and the run still holds its workspace as the writer. + */ +export function isSupersededWorkspaceExport(coordinator: Coordinator) { + const detail = coordinator.failureDetail; + return Boolean(coordinator.resultId) && coordinator.phase === "terminal_failure" && coordinator.failureCode === SUPERSEDED + && typeof detail?.workspaceFinalizeAttempt === "number"; +} + +/** + * Names the retained-file hold of a superseded export, so that a spent + * re-export budget continues the task from its last save (the retained-source + * fallback) instead of waiting for a person: no result waits for these files. + */ +export async function markSupersededWorkspaceExportHold(db: Db, run: Pick) { + const marker = { schema: SUPERSEDED_WORKSPACE_EXPORT_SCHEMA, runId: run.id }; + await db.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} + || ${JSON.stringify({ [SUPERSEDED_WORKSPACE_EXPORT_KEY]: marker })}::jsonb` }) + .where(and(eq(issueRecoveryActions.companyId, run.companyId), + eq(issueRecoveryActions.fingerprint, `native-workspace:${run.id}`), + sql`${issueRecoveryActions.evidence}->>'runId' = ${run.id}`, + sql`${issueRecoveryActions.evidence}->'workspaceRestoreRecovery' is not null`, + sql`not (${issueRecoveryActions.evidence} ? ${SUPERSEDED_WORKSPACE_EXPORT_KEY})`)); +} + +/** + * An ended run whose superseded export left its files unsaved and its + * workspace held, with no retained-file obligation recorded yet. Rows written + * by releases before this rule match too, so the new build settles them itself. + */ +export function supersededWorkspaceExportUnpreservedCondition() { + return and(eq(nativeRunFinalizations.phase, "terminal_failure"), eq(nativeRunFinalizations.failureCode, SUPERSEDED), + isNotNull(nativeRunFinalizations.resultId), isNull(nativeRunFinalizations.leaseOwner), + sql`${nativeRunFinalizations.failureDetail} ? 'workspaceFinalizeAttempt'`, + eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.status, "failed"), + sql`${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,state}' = 'prepared'`, + sql`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync' is null`, + sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb)->'workspaceRestoreRecovery' is null`, + sql`exists (select 1 from execution_workspaces workspace where workspace.company_id = ${heartbeatRuns.companyId} + and workspace.id::text = ${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,workspaceId}' + and workspace.metadata#>>'{remoteWorkspace,writerRunId}' = ${heartbeatRuns.id}::text + and workspace.metadata#>>'{remoteWorkspace,dirty}' = 'true')`); +} + +/** + * Records the retained-file obligation of an ended superseded export (the + * system workspace recovery then re-exports it from the retained sandbox). + * Returns when to check again: only while a just-ended run's sandbox has not + * yet confirmed its stop, the one precondition that arrives by itself. + */ +export async function preserveSupersededWorkspaceExport(db: Db, runId: string, + preserve: (db: Db, run: typeof heartbeatRuns.$inferSelect) => Promise): Promise<{ recheckAt: number | null }> { + const [row] = await db.select({ run: heartbeatRuns }).from(nativeRunFinalizations) + .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, nativeRunFinalizations.runId), + eq(heartbeatRuns.companyId, nativeRunFinalizations.companyId))) + .where(and(eq(nativeRunFinalizations.runId, runId), supersededWorkspaceExportUnpreservedCondition())); + if (!row) return { recheckAt: null }; + const preserved = await preserve(db, row.run) as typeof heartbeatRuns.$inferSelect | undefined; + if (preserved && hasRequiredWorkspaceRecovery(preserved.resultJson)) return { recheckAt: null }; + const reference = remoteWorkspaceRunReferenceSchema.safeParse(row.run.runnerProfileJson?.remoteWorkspace); + if (!reference.success) return { recheckAt: null }; + const [lease] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, row.run.companyId), + eq(environmentLeases.id, reference.data.leaseId), eq(environmentLeases.heartbeatRunId, row.run.id), + inArray(environmentLeases.status, ["active", "pending_cleanup", "released"]))); + const ended = row.run.finishedAt?.getTime() ?? 0; + return { recheckAt: lease && !hasRemoteTerminationReceipt(lease) && Date.now() < ended + STOP_RECEIPT_WAIT_MS + ? Date.now() + STOP_RECEIPT_RECHECK_MS : null }; +} diff --git a/server/src/services/preserved-result-finalization.ts b/server/src/services/preserved-result-finalization.ts index c9ebcb50f95..95f7e31476b 100644 --- a/server/src/services/preserved-result-finalization.ts +++ b/server/src/services/preserved-result-finalization.ts @@ -60,8 +60,9 @@ async function readApplicableResult(db: Db, companyId: string, runId: string, lo const coordinatorQuery = db.select().from(nativeRunFinalizations).where(and( eq(nativeRunFinalizations.companyId, companyId), eq(nativeRunFinalizations.runId, runId))); const [coordinator] = lock ? await coordinatorQuery.for("update") : await coordinatorQuery; + // A result retired as superseded is never applied, even after its files are saved. if (!coordinator?.resultId || coordinator.phase !== "terminal_failure" || coordinator.leaseOwner || - coordinator.failureDetail?.successorRunId) return null; + coordinator.failureDetail?.successorRunId || coordinator.failureCode === "native_finalization_superseded") return null; const issueQuery = db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, coordinator.issueId))); const [issue] = lock === "nowait" ? await issueQuery.for("update", { noWait: true }) : lock ? await issueQuery.for("update") : await issueQuery; diff --git a/server/src/services/superseded-workspace-export-recovery.test.ts b/server/src/services/superseded-workspace-export-recovery.test.ts new file mode 100644 index 00000000000..03b65a6e329 --- /dev/null +++ b/server/src/services/superseded-workspace-export-recovery.test.ts @@ -0,0 +1,179 @@ +import { randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { eq } from "drizzle-orm"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { environmentLeases, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, + nativeRunResults } from "@paperclipai/db"; +import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; +import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js"; +import { readRemoteWorkspaceWriterHold } from "./remote-workspace-writer-hold.js"; +import { createSavedWorkspaceLoader } from "./remote-workspaces.js"; +import { prepareStoppedSandboxForRecovery, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js"; +import { readSystemRecovery } from "./system-recovery-state.js"; +import { adoptSystemRecovery } from "./system-recovery.js"; +import { RETAINED_SOURCE_FALLBACK_OUTCOME } from "./workspace-recovery-fallback.js"; +import { readWorkspaceRecovery } from "./workspace-recovery-receipt.js"; +import { recoverRetainedWorkspace } from "./workspace-recovery.js"; +import { resumePreservedResultFinalization } from "./preserved-result-finalization.js"; +import { acceptedRemoteExport, connectionString, db, run, useWorkspaceRecoveryFixture } from "./__tests__/workspace-recovery-fixture.js"; + +const targets = vi.hoisted(() => new Map()); +vi.mock("./environment-execution-target.js", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, resolveEnvironmentExecutionTarget: async (input: Parameters[0]) => + (input.leaseId ? targets.get(input.leaseId) : undefined) ?? actual.resolveEnvironmentExecutionTarget(input) }; +}); + +const workspace = async (id: string) => (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, id)))[0]!; +const adminText = /sandbox|environment|administrator|board member|owner must|inspect the run/i; + +describe("a superseded run whose workspace export failed", () => { + useWorkspaceRecoveryFixture(targets); + beforeEach(() => { + vi.stubEnv("DATABASE_URL", connectionString); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "63".repeat(32)); + }); + afterEach(() => { vi.unstubAllEnvs(); }); + + /** + * The staging shape: the accepted result's export failed once + * with remote_workspace_archive_fs_eacces, a newer run had already decided + * the task, so the finalizer was retired as superseded and the run ended + * failed. Its ephemeral sandbox was stopped and retained, and the run is + * still the dirty writer of the task's workspace. No retained-file + * obligation exists, so nothing ever settles that writer. + */ + async function supersededExport() { + const f = await acceptedRemoteExport(); + const now = new Date(); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null, leaseExpiresAt: null, + nextAttemptAt: null, failureCode: "native_finalization_superseded", failureDetail: { + message: "remote_workspace_archive_fs_eacces", originalFailureCode: "native_workspace_sync_out_failed", + failureCause: "remote_workspace_archive_fs_eacces", workspaceFinalizeAttempt: 1, + recoveryOwner: { kind: "none", reason: "newer_native_decision" } } }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + const source = await run(f.runId); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: now, controllerLeaseExpiresAt: null, + errorCode: "native_finalization_superseded", + error: "A newer run already decided this task, so this run's result was kept for reference and not applied.", + resultJson: { ...source.resultJson, failureCode: "native_finalization_superseded", + failureCause: "remote_workspace_archive_fs_eacces", originalFailureCode: "native_workspace_sync_out_failed", + finalizationPhase: "terminal_failure" } }).where(eq(heartbeatRuns.id, f.runId)); + // The retirement resolved the finalizer's own recovery action. + await db.update(issueRecoveryActions).set({ status: "resolved", outcome: "false_positive", resolvedAt: now, + cause: "native_workspace_sync_out_failed", evidence: { runId: f.runId } }) + .where(eq(issueRecoveryActions.id, f.actionId)); + await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, f.issueId)); + const [lease] = await db.update(environmentLeases).set({ leasePolicy: "ephemeral" }) + .where(eq(environmentLeases.id, f.lease.id)).returning(); + const stopping = (await prepareStoppedSandboxForRecovery(db, lease))!; + await settleStopOnlyCleanup(db, stopping, { attemptId: String(stopping.metadata!.pendingCleanupAttemptId), + receipt: { providerLeaseId: f.providerLeaseId, state: "stopped" } }); + return f; + } + + async function preservation(f: { companyId: string; runId: string }) { + const rows = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId)); + return rows.find(row => row.evidence.runId === f.runId && row.evidence.workspaceRestoreRecovery) ?? null; + } + + async function settleThroughProjectionWorker(f: { companyId: string; issueId: string; runId: string }) { + const work = nativeRunProjectionWork(db); + const entries = (await work.load({ companyId: f.companyId })).filter(entry => entry.id === f.runId); + for (const entry of entries) await work.check(entry); + return entries.length; + } + + it("records the files for automatic recovery, saves them from the retained sandbox and frees the workspace", async () => { + const f = await supersededExport(); + const [result] = await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.resultId)); + const [task] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + const authority = readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)!; + expect(authority).toMatchObject({ writerRunId: f.runId, dirty: true }); + + // The native run projection worker finds the ended run at startup and + // records the retained-file obligation the superseded retirement skipped. + expect(await settleThroughProjectionWorker(f)).toBe(1); + const hold = await preservation(f); + expect(hold).not.toBeNull(); + expect(hold!.evidence).toMatchObject({ supersededWorkspaceExport: { runId: f.runId } }); + expect((await run(f.runId)).resultJson).toMatchObject({ workspaceRestoreRecovery: { leaseIds: [f.lease.id] } }); + // A second pass finds nothing left to do. + expect(await settleThroughProjectionWorker(f)).toBe(0); + + // While Paperclip saves those files, the workspace waits for that save. + const held = await readRemoteWorkspaceWriterHold(db, { companyId: f.companyId, authority }); + expect(held).toMatchObject({ kind: "held", writer: { runId: f.runId, ended: true } }); + + // The system workspace recovery saves the files from the retained sandbox. + const input = { ...f.input, actionId: hold!.id }; + await adoptSystemRecovery(db, input); + await recoverRetainedWorkspace(input); + const recovered = (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, hold!.id)))[0]!; + expect(readWorkspaceRecovery(recovered.evidence)?.status).toBe("recovered"); + const saved = readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)!; + expect(saved).toMatchObject({ writerRunId: null, dirty: false }); + expect(await readRemoteWorkspaceWriterHold(db, { companyId: f.companyId, authority: saved })).toEqual({ kind: "free" }); + const loaded = await createSavedWorkspaceLoader(db)({ companyId: f.companyId, workspaceId: f.workspaceId, + snapshotId: saved.snapshotId! }); + try { expect(await readFile(path.join(loaded.root, "saved.txt"), "utf8")).toBe("finished work\n"); } + finally { await loaded.cleanup(); } + + // The superseded result is never applied, and the task's decision stands. + expect(await resumePreservedResultFinalization(db, f.companyId, f.runId)).toBeFalsy(); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]) + .toMatchObject({ phase: "terminal_failure", failureCode: "native_finalization_superseded", resultId: result.id }); + expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.resultId)))[0]).toEqual(result); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]).toMatchObject({ + status: task.status, lastStatusDecisionId: task.lastStatusDecisionId }); + }, 60_000); + + it("continues from the last save with the files attached when every re-export fails", async () => { + const f = await supersededExport(); + expect(await settleThroughProjectionWorker(f)).toBe(1); + const hold = (await preservation(f))!; + const input = { ...f.input, actionId: hold.id }; + await adoptSystemRecovery(db, input); + // The bounded re-export budget is spent: every save failed the same way. + const state = readSystemRecovery(hold.evidence) ?? readSystemRecovery((await db.select().from(issueRecoveryActions) + .where(eq(issueRecoveryActions.id, hold.id)))[0]!.evidence)!; + const current = (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, hold.id)))[0]!; + await db.update(issueRecoveryActions).set({ evidence: { ...current.evidence, systemRecovery: { ...state, + revision: state.revision + 6, phase: "preserve_files", state: "exhausted", attempts: 6, retriesUsed: 5, + attemptId: randomUUID(), firstFailureAt: Date.now() - 30 * 60_000, deadlineAt: Date.now(), nextAttemptAt: null, + failure: { code: "export_interrupted", retryClass: "retry_transient", evidenceFingerprint: "e".repeat(64) } } } }) + .where(eq(issueRecoveryActions.id, hold.id)); + + await recoverRetainedWorkspace(input); + + const settled = (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, hold.id)))[0]!; + expect(settled).toMatchObject({ status: "resolved", outcome: "restored", ownerType: "system", + resolutionNote: RETAINED_SOURCE_FALLBACK_OUTCOME }); + expect(settled.evidence.executionReconciliation).toMatchObject({ runId: f.runId, + retainedSourceFallback: { outcome: RETAINED_SOURCE_FALLBACK_OUTCOME, quarantine: { state: "captured" } } }); + // Plain words, no request for a person to fix anything. + const message = readWorkspaceRecovery(settled.evidence)?.message; + expect(message).toContain("could not save the previous run's unsaved files"); + for (const text of [settled.nextAction, message]) expect(String(text)).not.toMatch(adminText); + // Only this run's epoch is set aside; its location stays on record and + // the next run restores the last save into a new root. + const after = await workspace(f.workspaceId); + const authority = readRemoteWorkspaceAuthority(after.metadata)!; + expect(authority).toMatchObject({ writerRunId: null, dirty: false, location: null, epoch: 2 }); + expect(after.metadata?.remoteWorkspaceAbandonedEpochs).toEqual([expect.objectContaining({ epoch: 1, runId: f.runId, + location: expect.objectContaining({ leaseId: f.lease.id, providerLeaseId: f.providerLeaseId }) })]); + expect(await readRemoteWorkspaceWriterHold(db, { companyId: f.companyId, authority })).toEqual({ kind: "free" }); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]) + .toMatchObject({ phase: "terminal_failure", failureCode: "native_finalization_superseded" }); + }, 60_000); + + it("leaves a superseded result alone when it never failed in the workspace export", async () => { + const f = await supersededExport(); + await db.update(nativeRunFinalizations).set({ failureDetail: { originalFailureCode: "native_status_conflict", + recoveryOwner: { kind: "none", reason: "newer_native_decision" } } }).where(eq(nativeRunFinalizations.runId, f.runId)); + expect(await settleThroughProjectionWorker(f)).toBe(0); + expect(await preservation(f)).toBeNull(); + }, 60_000); +}); diff --git a/server/src/services/workspace-recovery-fallback-discovery.ts b/server/src/services/workspace-recovery-fallback-discovery.ts index 2a8d97553d8..8fff7463d39 100644 --- a/server/src/services/workspace-recovery-fallback-discovery.ts +++ b/server/src/services/workspace-recovery-fallback-discovery.ts @@ -15,6 +15,11 @@ export const EXPORT_HOLD_CAUSES = ["native_workspace_sync_out_unrecoverable", "n * no-replay hold. It is recorded resolved-but-blocked: replay stays blocked * until its files are recovered, or the task continues from the last save. */ export const NATIVE_NO_REPLAY_HOLD_CAUSE = "native_continuation_requires_reconciliation"; +/** Marks the retained-file hold of a result retired as superseded while its + * workspace export failed. No result waits for those files, so once the + * bounded re-export is spent the task continues from its last save. */ +export const SUPERSEDED_WORKSPACE_EXPORT_KEY = "supersededWorkspaceExport"; +export const SUPERSEDED_WORKSPACE_EXPORT_SCHEMA = "paperclip.superseded-workspace-export.v1"; type HoldAction = { cause: string; status: string; outcome: string | null; evidence: Record }; const list = (values: readonly string[]) => sql.join(values.map(value => sql`${value}`), sql`, `); @@ -41,10 +46,19 @@ function retainedFileHoldCondition() { and ${issueRecoveryActions.evidence}#>>'{automaticRecovery,runId}' = ${issueRecoveryActions.evidence}->>'runId'))`; } +/** Whether a hold names the superseded export of its own source run. */ +export function isSupersededWorkspaceExportHold(evidence: Record | null | undefined) { + const marker = evidence?.[SUPERSEDED_WORKSPACE_EXPORT_KEY]; + return field(marker, "schema") === SUPERSEDED_WORKSPACE_EXPORT_SCHEMA && typeof evidence?.runId === "string" + && field(marker, "runId") === evidence.runId; +} + /** Whether an exhausted hold may continue from the last save: its source is - * refused permanently, or its stop stayed unproven through every re-arm. */ + * refused permanently, its stop stayed unproven through every re-arm, or it + * holds the files of a superseded result whose bounded re-export is spent. */ export function retainedHoldFallbackEligible(evidence: Record | null | undefined, state: SystemRecoveryState | null) { if (!state || state.state !== "exhausted") return false; + if (isSupersededWorkspaceExportHold(evidence)) return true; const failure = state.failure; if (failure && (UNVERIFIABLE_SOURCE_CODES as readonly string[]).includes(failure.code) && (PERMANENT_SOURCE_RETRY_CLASSES as readonly string[]).includes(failure.retryClass)) return true; @@ -160,6 +174,12 @@ export function retainedSourceFallbackPendingCondition() { and exists (select 1 from heartbeat_runs source where source.company_id = ${issueRecoveryActions.companyId} and source.id = ${sourceRunId} and source.runner_profile_json#>>'{remoteWorkspace,state}' = 'prepared')) + or (${retainedFileHoldCondition()} + and ${issueRecoveryActions.evidence}#>>'{${sql.raw(SUPERSEDED_WORKSPACE_EXPORT_KEY)},schema}' = ${SUPERSEDED_WORKSPACE_EXPORT_SCHEMA} + and ${issueRecoveryActions.evidence}#>>'{${sql.raw(SUPERSEDED_WORKSPACE_EXPORT_KEY)},runId}' = ${issueRecoveryActions.evidence}->>'runId' + and exists (select 1 from heartbeat_runs source where source.company_id = ${issueRecoveryActions.companyId} + and source.id = ${sourceRunId} + and source.runner_profile_json#>>'{remoteWorkspace,state}' = 'prepared')) or (${retainedFileHoldCondition()} and ${exhaustedStopProofWait} and ${stopProofRearms} >= ${STOP_PROOF_MAX_REARMS} and exists (select 1 from heartbeat_runs source where source.company_id = ${issueRecoveryActions.companyId} and source.id = ${sourceRunId} diff --git a/server/src/services/workspace-recovery-fallback.ts b/server/src/services/workspace-recovery-fallback.ts index 2374117adf1..cde266a2247 100644 --- a/server/src/services/workspace-recovery-fallback.ts +++ b/server/src/services/workspace-recovery-fallback.ts @@ -27,7 +27,7 @@ import { hasStopOnlyCleanup, prepareHistoricalSandboxStopVerification, prepareSa import { holdOwnsBlockedStatus } from "./recovery-hold-blocked-status.js"; import { withNativeWorkspaceFinalizationOwnership } from "./native-runtime/native-workspace-finalization-ownership.js"; import { NATIVE_NO_REPLAY_HOLD_CAUSE, PERMANENT_SOURCE_RETRY_CLASSES, RETAINED_SOURCE_FALLBACK_CHECK_KEY, UNVERIFIABLE_SOURCE_CODES, isRetainedFileHold, - retainedHoldFallbackEligible } from "./workspace-recovery-fallback-discovery.js"; + isSupersededWorkspaceExportHold, retainedHoldFallbackEligible } from "./workspace-recovery-fallback-discovery.js"; import { captureReadOnlyWorkspaceArchive, sandboxProviderFailureLabel } from "./remote-workspace-transport.js"; import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; import { readSystemRecovery, type SystemRecoveryState } from "./system-recovery-state.js"; @@ -348,9 +348,11 @@ export async function continueFromLastSavedWorkspace(input: Source & { // decision owns the block: it is kept and no continuation is started. const ownsBlock = holdOwnsBlockedStatus(action, issue, run); const dispatchable = issue.status !== "blocked" || ownsBlock; + // A superseded export's files were not unverifiable: saving them kept failing. + const could = isSupersededWorkspaceExportHold(action.evidence) ? "could not save" : "could not verify"; const userText = quarantine.state === "captured" - ? "Paperclip could not verify the previous run's unsaved files, so the task continues from the last saved version. Those files are attached to the task, unapplied." - : "Paperclip could not verify the previous run's unsaved files, so the task continues from the last saved version. Those files could not be read."; + ? `Paperclip ${could} the previous run's unsaved files, so the task continues from the last saved version. Those files are attached to the task, unapplied.` + : `Paperclip ${could} the previous run's unsaved files, so the task continues from the last saved version. Those files could not be read.`; const evidence = { ...action.evidence, automaticRecovery: undefined, [RETAINED_SOURCE_FALLBACK_CHECK_KEY]: undefined, retainedSourceFallback: { ...fallback, failureCode: current.system.failure?.code ?? null, recordedAt: now.toISOString() }, workspaceRecovery: { ...(readWorkspaceRecovery(action.evidence) ?? {}), message: userText, retryable: false },