diff --git a/server/src/__tests__/native-result-commit-work.test.ts b/server/src/__tests__/native-result-commit-work.test.ts index 25edb2561e1..0390eaca819 100644 --- a/server/src/__tests__/native-result-commit-work.test.ts +++ b/server/src/__tests__/native-result-commit-work.test.ts @@ -16,6 +16,7 @@ import { agents, companies, completionContracts, createDb, heartbeatRuns, issues statusDecisionEffects, statusDecisions, workspaceOperations, type Db } from "@paperclipai/db"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { nativeResultCommitWork } from "../services/native-runtime/native-result-commit-work.js"; +import { reserveLiveNativeFinalization } from "../services/native-runtime/live-native-finalization.js"; import { finalizeNativeRun, NativeFinalizationDeferredError } from "../services/native-runtime/native-run-finalizer.js"; import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js"; import { NativeRunCoordinatorStore } from "../services/native-runtime/native-run-coordinator-store.js"; @@ -137,6 +138,22 @@ describe("accepted result commit lifecycle", () => { await quiet(); await coordinator!.stop(); await start(); await flush(); await quiet(); expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, binding.runId))).toHaveLength(1); }); + it.each(["delivered", "lost"] as const)("leaves a live heartbeat's run to it and commits what it leaves behind (release notification %s)", async notification => { + await start(); await flush(); + const binding = await seed(); + const release = reserveLiveNativeFinalization(db, binding); + await complete(binding, "control_plane"); await barrier(binding); + await vi.advanceTimersByTimeAsync(5000); await delay(10); + // The reconciliation pass also leaves the reserved run to its heartbeat. + await reconcileNativeFinalizations(db, [binding.runId]); + expect((await state(binding.runId)).phase).not.toBe("committed"); + expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, binding.runId))).toHaveLength(0); + if (notification === "lost") vi.spyOn(db, "transaction").mockRejectedValueOnce(new Error("notification transaction failed")); + await release(); await release(); + if (notification === "lost") await vi.advanceTimersByTimeAsync(30_000); + await flush(); + expect((await state(binding.runId)).phase).toBe("committed"); await quiet(); + }); it("ignores routine company-only finalization hints and still commits accepted work", async () => { await start(); await flush(); await quiet(); const previous = db.$client.options.debug; diff --git a/server/src/__tests__/native-workspace-finalization-work.test.ts b/server/src/__tests__/native-workspace-finalization-work.test.ts index efa36ec0ef1..010e5eea476 100644 --- a/server/src/__tests__/native-workspace-finalization-work.test.ts +++ b/server/src/__tests__/native-workspace-finalization-work.test.ts @@ -16,6 +16,7 @@ import { nativeWorkspaceFinalizationWork } from "../services/native-runtime/nati import { attemptNativeWorkspaceFinalization } from "../services/native-runtime/native-workspace-finalizer.js"; import { getRunExecutionTraceContext, measureRunPhase } from "../services/execution-preparation-timing.js"; import { withNativeWorkspaceFinalizationOwnership } from "../services/native-runtime/native-workspace-finalization-ownership.js"; +import { reserveLiveNativeFinalization } from "../services/native-runtime/live-native-finalization.js"; import { resumeNativeWorkspaceAfterOwnerStop } from "../services/native-runtime/native-workspace-owner-recovery.js"; import * as platformProof from "../services/native-runtime/native-workspace-platform-proof.js"; import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js"; @@ -320,6 +321,22 @@ describe("workspace finalization scheduled lifecycle", () => { expect([...entries].sort((a, b) => a.dueAt - b.dueAt).map(item => item.id)) .toEqual([overdue.runId, interrupted.runId]); }); + it("leaves a result posted mid-run to the live heartbeat, then resumes after its release", async () => { + await start(); await flush(); + const binding = await seed(); + const release = reserveLiveNativeFinalization(db, binding); + const nested = reserveLiveNativeFinalization(db, binding); + await nested(); await nested(); + // The runner posts its accepted result before its process returns. + await complete(binding); await flush(); await quiet(); + expect(await successful(binding.runId)).toHaveLength(0); + const heartbeat = await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: binding.runId, liveHeartbeat: true }, + async () => "owned"); + expect(heartbeat).toEqual({ acquired: true, value: "owned" }); + // A heartbeat that stopped before exporting leaves the work to the worker. + await release(); await flush(); + expect(await successful(binding.runId)).toHaveLength(1); await quiet(); + }); it("waits quietly for the actual local owner to release", async () => { const binding = await seed(); await complete(binding); await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: binding.runId }, async () => { diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.ts index 746dcca88fa..ca9a10993f8 100644 --- a/server/src/services/heartbeat/runtime-execution.test.ts +++ b/server/src/services/heartbeat/runtime-execution.test.ts @@ -36,6 +36,9 @@ import { createHeartbeatRunState } from "./run-state.js"; import { buildEffectiveRunSessionConfigMetadata } from "./workspaces.js"; import { revokeHeartbeatRunGatewayTokens } from "./run-preparation.js"; import { executeHeartbeatRuntime, type HeartbeatRuntimeExecutionInput } from "./runtime-execution.js"; +import { liveNativeFinalizationReserved } from "../native-runtime/live-native-finalization.js"; +import { onNativeWorkspaceFinalizationRelease } from "../native-workspace-finalization-notifications.js"; +import { withNativeWorkspaceFinalizationOwnership } from "../native-runtime/native-workspace-finalization-ownership.js"; import { prepareNativeHeartbeatRun } from "../native-runtime/prepare-native-run.js"; import { buildNativeCompletionContract } from "../native-runtime/completion-contracts.js"; @@ -412,6 +415,36 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = expect(input.instructions.releaseInstructionCopy).toHaveBeenCalledOnce(); } + it.each(["returns", "throws"] as const)("keeps recovery workers off the run until the live heartbeat ends (runner %s)", async outcome => { + const { input } = await fixture(true); + let reservedDuringDispatch = false; + let workerOwnership: unknown; + vi.mocked(executePaperclipNativeSession).mockImplementationOnce(async () => { + // A runner posts its accepted result here, before it returns. + reservedDuringDispatch = liveNativeFinalizationReserved(input.run.id); + workerOwnership = await withNativeWorkspaceFinalizationOwnership({ db, companyId: input.run.companyId, runId: input.run.id }, + async () => "taken"); + if (outcome === "throws") throw new Error("runner failed after dispatch"); + return { exitCode: 0, signal: null, timedOut: false }; + }); + const execution = executeHeartbeatRuntime(db, input); + if (outcome === "throws") await expect(execution).rejects.toThrow(); else await execution; + expect(reservedDuringDispatch).toBe(true); + expect(workerOwnership).toEqual({ acquired: false, reason: "owner_active" }); + expect(liveNativeFinalizationReserved(input.run.id)).toBe(false); + expectCleanup(input); + }); + + it("still cleans up when waking the workers after the live heartbeat throws", async () => { + const { input } = await fixture(true); + const stop = onNativeWorkspaceFinalizationRelease(db, () => { throw new Error("release listener failed"); }); + try { + await expect(executeHeartbeatRuntime(db, input)).resolves.toMatchObject({ dispatched: true }); + } finally { stop(); } + expect(liveNativeFinalizationReserved(input.run.id)).toBe(false); + expectCleanup(input); + }); + it("passes preparation renewal ownership to native dispatch", async () => { const { input } = await fixture(true); const acquired = vi.fn(async () => undefined); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index 5ff0cc20362..bcfdfc1bce8 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -60,6 +60,7 @@ import { heartbeatRuns, nativeRunFinalizations, workspaceOperations } from "@pap import { conflict } from "../../errors.js"; import { getStartupTraceContext } from "../../instrumentation.js"; import { logger } from "../../middleware/logger.js"; +import { reserveLiveNativeFinalization } from "../native-runtime/live-native-finalization.js"; import { buildNativeProviderEnvironment, executePaperclipNativeSession, @@ -547,6 +548,11 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe const runGoalControlRequestId = readNonEmptyString( context.goalControlRequestId, ); + // The runner posts its accepted result before it returns. Keep the recovery + // workers off this run until this heartbeat has finished it. + const releaseLiveNativeFinalization = nativeRuntimeResolution.kind === "native" + ? reserveLiveNativeFinalization(db, { companyId: run.companyId, runId: run.id, issueId: issueRef?.id }) + : undefined; try { if (nativeRuntimeResolution.kind === "native") { if (!nativeExecution || !nativeRunnerInstanceId) @@ -1090,7 +1096,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe }; if (nativeWorkspaceSync || remoteWorkspaceAuthority) { const owned = await measureRunPhase("execution.workspace_ownership", () => withNativeWorkspaceFinalizationOwnership({ - db, companyId: run.companyId, runId: run.id, + db, companyId: run.companyId, runId: run.id, liveHeartbeat: true, ...(remoteWorkspaceAuthority && nativeRuntimeResolution.kind !== "native" && persistedExecutionWorkspace ? { remoteWorkspaceId: persistedExecutionWorkspace.id } : {}), }, completeWorkspace)); @@ -1326,6 +1332,8 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe } throw adapterErr; } finally { + // Never throws, so a failed cleanup below cannot keep the workers waiting. + await releaseLiveNativeFinalization?.(); try { if (!preserveRunGatewayTokens) await revokeHeartbeatRunGatewayTokens({ db, diff --git a/server/src/services/native-runtime/live-native-finalization.ts b/server/src/services/native-runtime/live-native-finalization.ts new file mode 100644 index 00000000000..4ca16ffe340 --- /dev/null +++ b/server/src/services/native-runtime/live-native-finalization.ts @@ -0,0 +1,48 @@ +import type { Db } from "@paperclipai/db"; +import { notifyNativeResultCommit } from "../native-result-commit-notifications.js"; +import { notifyNativeWorkspaceFinalization, releaseNativeWorkspaceFinalization } from "../native-workspace-finalization-notifications.js"; +import { logger } from "../../middleware/logger.js"; + +// Runs whose live heartbeat in this process will export the workspace and +// commit the accepted result itself. A runner posts its result before its +// process returns, and that post wakes the recovery workers. Without this +// reservation a worker could take the run from the heartbeat that is still +// finishing it, and the run would lose its exit code, log summary and usage. +const reservedRuns = new Map(); + +export function liveNativeFinalizationReserved(runId: string): boolean { + return reservedRuns.has(runId); +} + +/** + * Hold a run for its live heartbeat. The returned release never throws. It + * wakes the workers for this run, so they resume anything the heartbeat left + * unfinished; the result worker also rechecks a reserved run on its own. A + * crashed process loses the reservation, and startup recovery takes over. + */ +export function reserveLiveNativeFinalization(db: Db, scope: { companyId: string; runId: string; issueId?: string }) { + reservedRuns.set(scope.runId, (reservedRuns.get(scope.runId) ?? 0) + 1); + let released = false; + return async () => { + if (released) return; + released = true; + const remaining = (reservedRuns.get(scope.runId) ?? 1) - 1; + if (remaining > 0) reservedRuns.set(scope.runId, remaining); + else reservedRuns.delete(scope.runId); + if (remaining > 0) return; + try { + await db.transaction(async (tx) => { + await notifyNativeWorkspaceFinalization(tx, scope); + await notifyNativeResultCommit(tx, scope); + }); + } catch (error) { + // Startup and periodic discovery still find the run. + logger.warn({ err: error, runId: scope.runId }, "native finalization workers could not be woken after the live heartbeat"); + } + try { + releaseNativeWorkspaceFinalization(db, scope); + } catch (error) { + logger.warn({ err: error, runId: scope.runId }, "native workspace finalization could not be woken after the live heartbeat"); + } + }; +} diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index 0f9d07e25b0..89b268f7f48 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -1,4 +1,5 @@ import { sameRunResumeClaimEntry } from "./scheduled-resume-continuation.js"; +import { liveNativeFinalizationReserved } from "./live-native-finalization.js"; import { appendBoundedRecoveryHistory } from "./native-restart-recovery.js"; import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; import { currentControllerProcessStamp } from "../controller-process-identity.js"; @@ -692,6 +693,8 @@ async function reconcileNativeFinalizationsPass( const { operation } = outcome; const workspaceFinalizeStatus = "succeeded" as const; if (options.skipNativeResultCommit && row.coordinatorPhase !== "committed") continue; + // The run's live heartbeat commits it; its release wakes the result worker. + if (row.coordinatorPhase !== "committed" && liveNativeFinalizationReserved(row.runId)) continue; try { const finalized = await finalizeNativeRun({ db, @@ -749,6 +752,8 @@ async function reconcileNativeFinalizationsPass( continue; } if (options.skipNativeResultCommit && row.coordinatorPhase !== "committed") continue; + // The run's live heartbeat commits it; its release wakes the result worker. + if (row.coordinatorPhase !== "committed" && liveNativeFinalizationReserved(row.runId)) continue; const replayDecision = resolveNativeReconciliationStatus({ facts: { canonicalReplay: true }, priorIssueStatus: row.issueStatus as NativeAuthoritativeIssueStatus, diff --git a/server/src/services/native-runtime/native-result-commit-work.ts b/server/src/services/native-runtime/native-result-commit-work.ts index 41750feb276..fa1f7cafa8a 100644 --- a/server/src/services/native-runtime/native-result-commit-work.ts +++ b/server/src/services/native-runtime/native-result-commit-work.ts @@ -9,6 +9,9 @@ import { pendingNativeStatusEffectsCondition } from "./native-pending-effect-wor import { nativeRunnerOwnershipNotHeldCondition } from "./native-runner-ownership.js"; import { finalizeNativeRun, NativeFinalizationDeferredError, recordNativeFinalizationFailure } from "./native-run-finalizer.js"; import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; +import { liveNativeFinalizationReserved } from "./live-native-finalization.js"; + +const LIVE_HEARTBEAT_RECHECK_MS = 30_000; type Entry = { id: string; companyId: string; issueId: string; dueAt: number }; @@ -80,6 +83,9 @@ export function nativeResultCommitWork(db: Db, finalize = finalizeNativeRun) { const [row] = await candidates(scope); if (!row) return null; const due = entry(row); + // The live heartbeat commits its own result. Its release wakes this queue; + // the recheck still finds the run if that notification is lost. + if (liveNativeFinalizationReserved(current.id)) return { ...due, dueAt: Math.max(due.dueAt, Date.now() + LIVE_HEARTBEAT_RECHECK_MS) }; if (due.dueAt > Date.now()) return due; const [run] = await db.select({ profile: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns) .where(and(eq(heartbeatRuns.companyId, current.companyId), eq(heartbeatRuns.id, current.id))); diff --git a/server/src/services/native-runtime/native-workspace-finalization-ownership.ts b/server/src/services/native-runtime/native-workspace-finalization-ownership.ts index d9bb5eb8efc..685ca4456e6 100644 --- a/server/src/services/native-runtime/native-workspace-finalization-ownership.ts +++ b/server/src/services/native-runtime/native-workspace-finalization-ownership.ts @@ -14,6 +14,7 @@ import { and, eq, inArray, or, sql } from "drizzle-orm"; import { executionWorkspaces, heartbeatRuns, issues, issueRecoveryActions, nativeRunFinalizations, workspaceOperations, withDedicatedDbConnection, type Db } from "@paperclipai/db"; import { currentNativeWorkspaceRecoveryResult } from "./native-workspace-recovery-binding.js"; import { withWorkspaceLockKeepalive } from "./native-workspace-lock-keepalive.js"; +import { liveNativeFinalizationReserved } from "./live-native-finalization.js"; import { currentNativeControllerIdentity, type NativeControllerIdentity, @@ -105,9 +106,13 @@ export function nativeWorkspaceFinalizationOwnerCanUpgrade(profile: Record( - input: { db: Db; companyId: string; runId: string; retainedRepairActionId?: string; remoteWorkspaceId?: string }, + input: { db: Db; companyId: string; runId: string; retainedRepairActionId?: string; remoteWorkspaceId?: string; + /** The run's own live heartbeat; every other caller yields to it. */ + liveHeartbeat?: boolean }, action: (ownership: NativeWorkspaceFinalizationOwnership) => Promise, ): Promise<{ acquired: false; reason: NativeWorkspaceFinalizationWaitReason } | { acquired: true; value: T }> { + // Its release wakes the workers again. + if (!input.liveHeartbeat && liveNativeFinalizationReserved(input.runId)) return { acquired: false, reason: "owner_active" }; const identity = await currentNativeControllerIdentity(); const processGroupId = await currentProcessGroupId(); const deployment = currentDeploymentIdentity();