diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 0ae90ed415a..f83ff2811ee 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1983,6 +1983,14 @@ ordinary Retry can continue. Finalized workspaces, mismatched sources and accepted results keep their existing settlement paths. Reassignment does not discard the stopped run's files or grant authority over the new assignee. +The deadline is a backstop, not the normal failure path. When native task +preparation fails after a fresh controller claim and before any provider turn, +the controller stops renewal and releases only that exact claim. Ordinary +failure settlement then retains the source and fails the run with its original +error, as it does for a failure before the claim. A restart-recovery claim, +startup cleanup, Stop, restart detach, accepted result or different owner keeps +its existing owner and settlement path. + For older failed native runs that lack this bookkeeping, a deliberate Retry registers preservation from the exact prepared workspace descriptor and assigns the existing bounded maintenance owner. A live finalization owner or mismatched diff --git a/server/src/services/native-runtime/native-execution-lease.ts b/server/src/services/native-runtime/native-execution-lease.ts index dc28b1ad9fe..4e17b04eccf 100644 --- a/server/src/services/native-runtime/native-execution-lease.ts +++ b/server/src/services/native-runtime/native-execution-lease.ts @@ -1,7 +1,7 @@ import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; import type { Db } from "@paperclipai/db"; import { heartbeatRuns, livenessDb, nativeRunFinalizations } from "@paperclipai/db"; -import { and, eq, gt, sql } from "drizzle-orm"; +import { and, eq, gt, isNull, sql } from "drizzle-orm"; import { currentNativeControllerIdentity, type NativeControllerIdentity } from "./native-restart-recovery.js"; import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js"; @@ -30,6 +30,24 @@ export async function assertNativeSessionPreparationAuthority(input: { if (!owned) throw new Error("native_session_lease_lost"); } +/** + * Preparation failed after a fresh claim and before any provider turn. Release + * only this exact claim, so ordinary failure settlement can retain the source + * as it does for failures before the claim. Any other owner or a result keeps it. + */ +export async function releaseUnstartedNativeExecutionClaim(input: { + db: Db; runId: string; companyId: string; issueId: string; leaseOwner: string; attempt: number; +}): Promise { + const released = await input.db.update(nativeRunFinalizations) + .set({ leaseOwner: null, leaseExpiresAt: null, updatedAt: new Date() }) + .where(and(eq(nativeRunFinalizations.runId, input.runId), eq(nativeRunFinalizations.companyId, input.companyId), + eq(nativeRunFinalizations.issueId, input.issueId), eq(nativeRunFinalizations.leaseOwner, input.leaseOwner), + eq(nativeRunFinalizations.attempt, input.attempt), eq(nativeRunFinalizations.phase, "observed"), + isNull(nativeRunFinalizations.resultId))) + .returning({ runId: nativeRunFinalizations.runId }); + return released.length > 0; +} + export async function renewNativeSessionExecutionLease(input: { db: Db; runId: string; diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.test.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.test.ts index 6f94d461e24..e00347330c5 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.test.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.test.ts @@ -15,6 +15,7 @@ import { prepareSandboxStopAndRetain, settleStopOnlyCleanup } from "../sandbox-s import { preserveCancelledNativeWorkspaces, preserveNativeFailedWorkspace, preserveOrphanedNativeWorkspace, registerCancelledWorkspacePreservationWork } from "./native-failed-workspace-recovery.js"; import { claimNativeRestartRecoveries, currentNativeControllerIdentity, NATIVE_RESTART_RECOVERY_FAILED, type NativeRestartRecoveryClaim } from "./native-restart-recovery.js"; import { NativeWorkspaceFinalizationBusyError } from "./native-workspace-finalization-ownership.js"; +import { releaseUnstartedNativeExecutionClaim } from "./native-execution-lease.js"; import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js"; import { claimNativeSessionResumptions } from "./native-finalization-reconciler.js"; @@ -779,6 +780,33 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes expect(await readRun(f.runId)).toMatchObject({ status: "running", nativePhase: "observed" }); }); + it.each(["own", "foreign", "attempt", "accepted"])("retains files after an unstarted preparation claim is released: %s", async kind => { + const f = await seed(); + const leaseOwner = randomUUID(); + await db.update(heartbeatRuns).set({ status: "running", nativePhase: "observed", errorCode: null }) + .where(eq(heartbeatRuns.id, f.runId)); + await db.update(nativeRunFinalizations).set({ phase: "observed", attempt: 1, leaseOwner, + leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId)); + // The heartbeat cannot tell a finished preparation claim from a live owner. + await expect(preserveNativeFailedWorkspace(db, await readRun(f.runId), { allowPreterminal: true })) + .rejects.toBeInstanceOf(NativeWorkspaceFinalizationBusyError); + if (kind === "accepted") await db.update(nativeRunFinalizations).set({ phase: "result_accepted" }).where(eq(nativeRunFinalizations.runId, f.runId)); + const released = await releaseUnstartedNativeExecutionClaim({ db, runId: f.runId, companyId: f.companyId, issueId: f.issueId, + leaseOwner: kind === "foreign" ? randomUUID() : leaseOwner, attempt: kind === "attempt" ? 2 : 1 }); + expect(released).toBe(kind === "own"); + if (kind !== "own") { + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0].leaseOwner).toBe(leaseOwner); + expect(await readLease(f.lease.id)).toMatchObject({ status: "active" }); + return; + } + await preserveNativeFailedWorkspace(db, await readRun(f.runId), { allowPreterminal: true }); + expect(await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId))) + .toMatchObject([{ phase: "terminal_failure", leaseOwner: null, recoveryState: "blocked" }]); + expect(await readRun(f.runId)).toMatchObject({ status: "running", nativePhase: "terminal_failure", + resultJson: { workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: { leaseIds: [f.lease.id] } } }); + expect(await readLease(f.lease.id)).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure" }); + }); + it("refuses cleanup when another execution still owns the coordinator", async () => { const f = await seed(); await db.update(nativeRunFinalizations).set({ phase: "observed", leaseOwner: randomUUID() }) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index b7ced95d40c..440341a6d13 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -11791,6 +11791,43 @@ describe("native session bounded recovery", () => { } finally { release(); await running; renew.mockRestore(); } }); + it.each([false, true])("releases only a fresh claim after preparation fails before provider work (restart claim: %s)", async restarted => { + const fresh = { ...structuredClone(execution), provider: { kind: "acpx", agent: "claude", + model: "claude-sonnet-4-6", profile: QUALIFIED_ACPX_PROFILES.claude } } as NativeExecutionInput; + const updates: Array<{ table: unknown; values: Record }> = []; + const { currentNativeControllerIdentity } = await import("./native-restart-recovery.js"); + const controller = await currentNativeControllerIdentity(); + const restartRecovery = restarted ? { kind: "reattach_remote_runner", runId: fresh.binding.runId, leaseOwner: "restart-owner", + controllerGeneration: 2, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null, + remote: { providerLeaseId: "fresh-provider-lease", remoteCwd: "/workspace" } } as const : undefined; + const db = leaseDb(fresh, restarted ? { leaseOwner: "restart-owner", controllerBootId: controller.bootId, + controllerPid: controller.pid, controllerGeneration: 2, attempt: 0 } : {}, {}, updates); + // Fails inside task preparation, after the claim commits and renewal starts. + const failure = new Error("native_harness_storage_owner_missing"); + const order: string[] = []; + const stop = vi.fn(async () => { order.push("stop"); }); + const renew = vi.spyOn(nativeLeaseRenewal, "startNativeSessionExecutionLeaseRenewal").mockReturnValue({ stop, remainingMs: () => 60_000 }); + const release = vi.spyOn(nativeLeaseRenewal, "releaseUnstartedNativeExecutionClaim") + .mockImplementation(async () => { order.push("release"); return true; }); + try { + await expect(executePaperclipNativeSession({ db, execution: fresh, runnerInstanceId: "fresh-runner", + onExecutionLeaseAcquired: async () => { throw failure; }, + ...(restartRecovery ? { leaseOwner: restartRecovery.leaseOwner, restartRecovery } : {}), + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", environmentId: "fresh-environment", leaseId: "fresh-lease" }, + })).rejects.toBe(failure); + expect(updates.some(update => update.table === nativeRunFinalizations && "phase" in update.values + && update.values.phase !== "observed")).toBe(false); + if (restarted) { + // A restart claim keeps its exact recovery fence for the heartbeat's retention check. + expect(release).not.toHaveBeenCalled(); + } else { + expect(release).toHaveBeenCalledExactlyOnceWith({ db, runId: fresh.binding.runId, companyId: fresh.binding.companyId, + issueId: fresh.binding.issueId, leaseOwner: expect.stringMatching(/^fresh-runner:/), attempt: 1 }); + expect(order).toEqual(["stop", "release"]); + } + } finally { renew.mockRestore(); release.mockRestore(); } + }); + it("persists closed remote recovery evidence without replacing saved run results", async () => { const updates: Array<{ table: unknown; values: Record }> = []; const onEvent = vi.fn(async () => undefined); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index d36f7395977..e923de72117 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -29,7 +29,7 @@ import { providerPackWorkspaceId, recordAdmittedRemoteProviderPack, resolveRemot import { nativeProviderPackAdmissionPending, rememberAdmittedNativeProviderPack, rememberRecoveredNativeProviderPack, recordNativeProviderPackAdmission, type NativeProviderPackAdmissionSource } from "./native-provider-pack-admission.js"; import { type ProviderPackInstallation, verifyRemoteProviderPack as verifyRemoteProviderPackIdentity } from "./remote-provider-pack-verification.js"; import { writeEnvironmentCleanup } from "../environment-cleanup-notifications.js"; -import { NATIVE_SESSION_EXECUTION_LEASE_TTL_MS, startNativeSessionExecutionLeaseRenewal, type NativeExecutionLeaseRenewal } from "./native-execution-lease.js"; +import { NATIVE_SESSION_EXECUTION_LEASE_TTL_MS, releaseUnstartedNativeExecutionClaim, startNativeSessionExecutionLeaseRenewal, type NativeExecutionLeaseRenewal } from "./native-execution-lease.js"; export { renewNativeSessionExecutionLease } from "./native-execution-lease.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { writeExecutionControl } from "../execution-control-notifications.js"; @@ -353,6 +353,8 @@ type NativeSessionStartup = { priorRestartDetach?: symbol; leaseOwner?: string; executionLeaseRenewal?: NativeExecutionLeaseRenewal; + /** A fresh claim whose preparation failed before any provider turn. */ + unstartedClaim?: { leaseOwner: string; attempt: number }; executionContext?: NativeExecutionContext; settled: Promise; settle: () => void; @@ -8607,6 +8609,12 @@ export async function executePaperclipNativeSession(input: { // Remove the settled startup before yielding, so detachment cannot reacquire // its old resolved session while renewal cleanup drains. await startup.executionLeaseRenewal?.stop().catch(() => undefined); + if (startup.unstartedClaim && !nativeRunsDetachingForRestart.has(runId)) { + await releaseUnstartedNativeExecutionClaim({ db: input.db, runId, companyId: input.execution.binding.companyId, + issueId: input.execution.binding.issueId, ...startup.unstartedClaim }).catch(error => { + logger.warn({ err: error, runId }, "Native preparation claim release failed; the control deadline will settle it"); + }); + } try { await cleanupStagedAttachments(); } catch (cleanupError) { @@ -8838,6 +8846,7 @@ async function executePaperclipNativeSessionWithinScope( let providerPackAdmissionSource: NativeProviderPackAdmissionSource | undefined; let confirmedLeaseDeadline = 0; let claimedStartupContext: NativeExecutionContext | undefined; + let freshClaimAttempt: number | undefined; try { attempt = await trace.measure( "native.coordinator.claim", @@ -9061,6 +9070,7 @@ async function executePaperclipNativeSessionWithinScope( const startup = nativeSessionStartups.get(input.execution.binding.runId); if (!startup) throw new Error("native_session_startup_owner_missing"); startup.leaseOwner = leaseOwner; + if (!input.restartRecovery) freshClaimAttempt = claimedAttempt; // Own renewal as soon as the claim commits, including slow tracing, // accounting history and retained-session cleanup before dispatch. startup.executionLeaseRenewal = startNativeSessionExecutionLeaseRenewal({ @@ -9106,6 +9116,17 @@ async function executePaperclipNativeSessionWithinScope( } catch (error) { await trace.end(taskPrepareScope, { outcome: "failed" }); await trace.finish("failed"); + // No provider turn started under this fresh claim. Leaving it held makes + // failure settlement treat this finished attempt as a live owner, so the + // run waits for the control deadline. The startup owner releases only this + // exact claim after renewal stops; a restart claim, startup cleanup, Stop or + // restart detach keeps its owner. + const startup = nativeSessionStartups.get(input.execution.binding.runId); + if (startup && freshClaimAttempt !== undefined && !claimedStartupContext + && !(error instanceof NativeCancellationPendingRecoveryError) + && !(error instanceof NativeControllerDetachedForRestartError)) { + startup.unstartedClaim = { leaseOwner, attempt: freshClaimAttempt }; + } throw error; } const controlPlaneInstanceId = `${effectiveRunnerInstanceId}:control`;