diff --git a/server/src/__tests__/assigned-work-invariant.test.ts b/server/src/__tests__/assigned-work-invariant.test.ts index 1fc494596d0..4261c4b1598 100644 --- a/server/src/__tests__/assigned-work-invariant.test.ts +++ b/server/src/__tests__/assigned-work-invariant.test.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql as sqlTag } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { enableReattachPhase, joinReattachPhase, resetReattachPhaseForTests, settleReattachRun } from "../services/heartbeat/run-admission.js"; import { activityLog, agents, agentTaskSessions, agentWakeupRequests, approvals, companies, createDb, heartbeatRuns, issueApprovals, issueRecoveryActions, issueRelations, issues, issueThreadInteractions, nativeRunFinalizations, @@ -367,6 +368,33 @@ describePostgres("assigned-work invariant sweep", () => { expect(queue.calls).toHaveLength(0); }, 60_000); + it("takes no action while handed-over live runs are being reached, and acts once the phase ends", async () => { + const f = await seedCompany(); + const otherBoot = randomUUID(); + const controllerProcess = { bootId: otherBoot, pid: 4242, hostHash: "host-b", processStartedAt: new Date(Date.now() - HOUR).toISOString() }; + const lapsed = await f.issue(); + await f.run(lapsed, { status: "running", errorCode: null, finishedAt: null, controllerBootId: otherBoot, controllerProcess, + controllerLeaseExpiresAt: new Date(Date.now() - 60_000) }); + const queue = wakeQueue(); + const settle = vi.fn(async () => undefined); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup, settleLostRunProcess: settle, + proveOwner: async () => ({ verdict: "gone", reason: "controller_stopped" }) }); + try { + enableReattachPhase(180_000); + joinReattachPhase([{ runId: "handed-over-live-run", reason: "handed_over", claimableAt: Date.now() }]); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: lapsed })) + .toMatchObject({ outcome: "deferred_reattach_phase", verdict: { reason: "run_owner_gone" } }); + expect(settle).not.toHaveBeenCalled(); + expect(queue.calls).toHaveLength(0); + settleReattachRun("handed-over-live-run", "reached"); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: lapsed })) + .toMatchObject({ outcome: "settled_lost_run", verdict: { reason: "run_owner_gone" } }); + expect(settle).toHaveBeenCalledTimes(1); + } finally { + resetReattachPhaseForTests(); + } + }, 60_000); + it("leaves tasks waiting on an approval, a question, a dependency or a declared unblock action", async () => { const f = await seedCompany(); const approvalTask = await f.issue(); diff --git a/server/src/__tests__/heartbeat-native-early-hand-off.test.ts b/server/src/__tests__/heartbeat-native-early-hand-off.test.ts new file mode 100644 index 00000000000..79c31c8bfa5 --- /dev/null +++ b/server/src/__tests__/heartbeat-native-early-hand-off.test.ts @@ -0,0 +1,115 @@ +import { and, eq, inArray } from "drizzle-orm"; +import { heartbeatRunEvents, heartbeatRuns, nativeRunFinalizations } from "@paperclipai/db"; +import { afterEach, expect, it, vi } from "vitest"; +import { heartbeatService } from "../services/heartbeat.js"; +import { currentControllerBootId } from "../services/controller-process-identity.js"; +import { claimNativeRestartRecoveries, evaluateNativeControllerTakeover } from "../services/native-runtime/native-restart-recovery.js"; +import { isRunHandedOffForShutdown } from "../services/heartbeat/shutdown-handoff.js"; +import { REMOTE_CONTROLLER_LEASE_MS } from "../services/native-runtime/native-runner-recovery-contract.js"; +import { describeEmbeddedPostgres, nativeRestartFixture } from "./helpers/native-restart.js"; +import { registerReattachPhaseWork, stopReattachWatch } from "../services/heartbeat/reattach-phase.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { isReattachPhaseActive, pendingReattachRunIds, resetReattachPhaseForTests, settleReattachRun, type ReattachPhaseSummary } from "../services/heartbeat/run-admission.js"; + +describeEmbeddedPostgres("an outgoing server hands live remote runs over at shutdown start", () => { + const fixture = nativeRestartFixture(); + const timings: Record = {}; + + async function seedLiveRemoteRuns(label: string, count: number) { + const seeded = []; + for (let index = 0; index < count; index += 1) seeded.push(await fixture.seedRun(`${label}-${index}`)); + const db = seeded[0]!.db; + const runIds = seeded.map(run => run.runId); + await db.update(nativeRunFinalizations).set({ + leaseOwner: "native-execution:outgoing", leaseExpiresAt: new Date(Date.now() + REMOTE_CONTROLLER_LEASE_MS), + controllerBootId: currentControllerBootId(), controllerPid: 2_147_000_000, controllerHostHash: "outgoing-container", + controllerProcessStartedAt: new Date(Date.now() - 600_000), + }).where(inArray(nativeRunFinalizations.runId, runIds)); + return { db, runIds }; + } + + async function incomingTakeoverReasons(db: Awaited>["db"], runIds: string[]) { + const owners = await db.select().from(nativeRunFinalizations).where(inArray(nativeRunFinalizations.runId, runIds)); + const now = new Date(); + return Promise.all(owners.map(async owner => (await evaluateNativeControllerTakeover({ owner, now, remoteLeaseFenced: true })).reason)); + } + + it.each([1, 20, 40])("releases the remote controller lease of %i live runs at once, so the incoming server may claim them", async (count) => { + const { db, runIds } = await seedLiveRemoteRuns(`HANDOFF${count}`, count); + expect(new Set(await incomingTakeoverReasons(db, runIds))).toEqual(new Set(["remote_controller_lease_live"])); + const outgoing = heartbeatService(db); + const startedAt = performance.now(); + const handedOff = await outgoing.handOffNativeRunsForShutdown("SIGTERM"); + timings[count] = Math.round(performance.now() - startedAt); + console.info(`[reattach-first] early hand-off of ${count} live runs: ${timings[count]} ms`); + expect(handedOff.sort()).toEqual([...runIds].sort()); + expect(new Set(await incomingTakeoverReasons(db, runIds))).toEqual(new Set(["remote_controller_lease_released_or_expired"])); + const owners = await db.select({ recoveryState: nativeRunFinalizations.recoveryState }).from(nativeRunFinalizations) + .where(inArray(nativeRunFinalizations.runId, runIds)); + expect(owners.every(owner => owner.recoveryState === "awaiting_runner_reattach")).toBe(true); + + const drained = await outgoing.drainRunningRunsForShutdown("SIGTERM", new Date(), runIds); + expect(drained.restartSuspendedRunIds.sort()).toEqual([...runIds].sort()); + const suspensions = await db.select({ runId: heartbeatRunEvents.runId }).from(heartbeatRunEvents).where(and( + inArray(heartbeatRunEvents.runId, runIds), eq(heartbeatRunEvents.eventType, "native.recovery.transition"))); + expect(suspensions).toHaveLength(count); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(inArray(heartbeatRuns.id, runIds)); + }, 300_000); + + it("never lets the outgoing server claim back a run it handed off, while the incoming server may take it over", async () => { + const { db, runIds } = await seedLiveRemoteRuns("RECLAIM", 1); + try { + const outgoing = heartbeatService(db); + expect(await outgoing.handOffNativeRunsForShutdown("SIGTERM")).toEqual(runIds); + await db.update(nativeRunFinalizations).set({ controllerPid: process.pid }).where(inArray(nativeRunFinalizations.runId, runIds)); + const outgoingPass = await claimNativeRestartRecoveries({ db, handedOffForShutdown: isRunHandedOffForShutdown, handedOffByThisController: isRunHandedOffForShutdown, + runIds, restartKind: "graceful" }); + expect(outgoingPass).toEqual([expect.objectContaining({ kind: "blocked", runId: runIds[0], reason: "handed_off_to_next_controller" })]); + const [owner] = await db.select({ leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, runIds[0]!)); + expect(owner?.leaseOwner).toBe("native-execution:outgoing"); + const incomingPass = await claimNativeRestartRecoveries({ db, controller: fixture.current.successor, runIds, restartKind: "graceful" }); + expect(incomingPass).toEqual([expect.not.objectContaining({ reason: "handed_off_to_next_controller" })]); + expect(await incomingTakeoverReasons(db, runIds)).toEqual(["remote_controller_lease_released_or_expired"]); + } finally { + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(inArray(heartbeatRuns.id, runIds)); + } + }, 120_000); + + it("leaves a run another server already took over alone", async () => { + const { db, runIds } = await seedLiveRemoteRuns("TAKEN", 1); + await db.update(nativeRunFinalizations).set({ controllerBootId: "incoming-boot" }) + .where(inArray(nativeRunFinalizations.runId, runIds)); + expect(await heartbeatService(db).handOffNativeRunsForShutdown("SIGTERM")).not.toContain(runIds[0]); + expect(await incomingTakeoverReasons(db, runIds)).toEqual(["remote_controller_lease_live"]); + }); + + let coordinator: ReturnType | undefined; + + afterEach(async () => { + await stopReattachWatch(); + await coordinator?.stop(); + coordinator = undefined; + resetReattachPhaseForTests(); + }); + + it("holds the incoming server's queued starts only from the hand-over, not while the outgoing server still drives its runs", async () => { + const { db, runIds } = await seedLiveRemoteRuns("PHASE", 3); + const ended: ReattachPhaseSummary[] = []; + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: () => undefined }); + await registerReattachPhaseWork(coordinator, db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20, + onEnd: summary => ended.push(summary) })?.ready; + await new Promise(resolve => setTimeout(resolve, 150)); + expect(isReattachPhaseActive()).toBe(false); + const handedOffAt = Date.now(); + expect((await heartbeatService(db).handOffNativeRunsForShutdown("SIGTERM")).sort()).toEqual([...runIds].sort()); + await vi.waitFor(() => expect(pendingReattachRunIds().sort()).toEqual([...runIds].sort()), { timeout: 5_000, interval: 10 }); + expect(isReattachPhaseActive()).toBe(true); + for (const runId of runIds) settleReattachRun(runId, "reached"); + expect(isReattachPhaseActive()).toBe(false); + const phase = ended.find(summary => summary.reason === "all_settled")!; + expect(phase).toMatchObject({ startReason: "handed_over", liveRuns: 3, reached: 3, gatedStarts: true }); + expect(phase.durationMs).toBeLessThan(Date.now() - handedOffAt + 50); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(inArray(heartbeatRuns.id, runIds)); + }, 60_000); +}); diff --git a/server/src/index.ts b/server/src/index.ts index 2998a733589..a7cea216ccf 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,6 +1,7 @@ import { tailscaleNetworks } from "./services/connectors/tailscale/network.js"; import type { IdleDeadlineOwners } from "./services/idle-deadline-types.js"; import { closeProcessRunAdmission, isProcessRunAdmissionClosed } from "./services/heartbeat/run-admission.js"; +import { registerReattachPhaseWork } from "./services/heartbeat/reattach-phase.js"; import { createRunStartHandover, registerRunStartHandover } from "./services/run-start-handover.js"; import { registerOwnershipReconciliationWork } from "./services/native-runtime/native-finalization-reconciler.js"; import { bindIdleDeadlineOwner } from "./services/idle-deadline-owner.js"; @@ -1299,6 +1300,7 @@ async function startServerWithDatabaseTeardown( runIds?: readonly string[] | null, ) => Promise) | null = null; let drainHeartbeatExecutionFinalizers: (() => Promise) | null = null; + let handOffNativeRunsForShutdown: ((signal: "SIGINT" | "SIGTERM") => Promise) | null = null; let prepareHotRestartShutdown: ((signal: "SIGINT" | "SIGTERM") => Promise<{ skipDrain: boolean; drainRunIds?: string[]; @@ -1343,6 +1345,18 @@ async function startServerWithDatabaseTeardown( // instead of behind every unrelated startup recovery pass. Agents with a run // left `running`, or a queued retry/continuation, keep waiting for the full // queued-run pass registered after native restart recovery below. + if (heartbeat && config.heartbeatSchedulerEnabled && !isWarmStandby()) { + const reattachHeartbeat = heartbeat; + registerReattachPhaseWork(deliveryWork, db as any, { + claim: runIds => reattachHeartbeat.recoverNativeRunsAfterRestart(new Date(), runIds), + onEnd: summary => { + if (!summary.gatedStarts || heartbeatSchedulerStopped) return; + trackHeartbeatSchedulerWork(accountingHeartbeat.resumeQueuedRunsAfterDrain().catch(err => { + logger.error({ err }, "failed to start queued runs held during the reattach phase"); + })); + }, + }); + } if (heartbeat) { const startupHeartbeat = heartbeat; trackHeartbeatSchedulerWork(timeStartupRecovery("fresh_queued_runs", async () => { @@ -1625,6 +1639,7 @@ async function startServerWithDatabaseTeardown( ); drainHeartbeatExecutionFinalizers = () => heartbeat.drainActiveRunExecutions(); + handOffNativeRunsForShutdown = signal => heartbeat.handOffNativeRunsForShutdown(signal); prepareHotRestartShutdown = heartbeat.prepareHotRestartShutdown; const routines = routineService(db as any, { pluginWorkerManager }); const statusCardEvaluation = statusCardEvaluationService(db as any, heartbeat); @@ -2199,6 +2214,13 @@ async function startServerWithDatabaseTeardown( const shutdownStartedAt = Date.now(); logger.info({ signal }, "shutdown signal received; starting ordered shutdown"); eventLoopStallMonitor.stop(); + const nativeHandOffStartedAt = Date.now(); + const nativeHandOff = runStartHandover && handOffNativeRunsForShutdown + ? handOffNativeRunsForShutdown(signal).then(runIds => { + logger.info({ signal, runs: runIds.length, durationMs: Date.now() - nativeHandOffStartedAt }, + "handed live native runs to the next server at shutdown start"); + }, err => { logger.warn({ err, signal }, "early native run hand-off failed; the graceful drain hands them off"); }) + : null; setStartupRecoveryPhase("draining"); for (const client of liveEventsWebSocketServer.clients) client.close(1012, "Server restarting"); liveEventsWebSocketServer.close(); @@ -2249,6 +2271,7 @@ async function startServerWithDatabaseTeardown( unregisterTerminalWorkspaceProbe?.(); unsubscribeNativeWorkspaceSyncRetention?.(); + if (nativeHandOff) await preDrain.run("native_run_hand_off", () => nativeHandOff); const prepareHotRestart = prepareHotRestartShutdown; const heartbeatShutdown = await coordinateHeartbeatSchedulerShutdown({ signal, diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 5260f9a1e46..46b1b53fa83 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -31,6 +31,7 @@ export const DELIVERY_QUEUES = { scheduledRetryDispatch: "scheduled-retry-dispatch", queuedRunDispatch: "queued-run-dispatch", runStartHandover: "run-start-handover", + reattachPhase: "reattach-phase", statusCardEvaluation: "status-card-evaluation", runEvidenceMigration: "run-evidence-migration", providerTraceRetention: "provider-trace-retention", diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index 2b1a7098c17..acc9a9132b7 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -37,6 +37,7 @@ import { isSupersededConversationRun } from "./agent-conversations.js"; import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { readSystemRecovery } from "./system-recovery-state.js"; +import { isReattachPhaseActive } from "./heartbeat/run-admission.js"; import { provenGoneLegacyControllerBootId } from "./legacy-controller-lease.js"; import { continueFreshInTransaction, continueHeldExecutions, freshContinuationCheckDue, recordFreshContinuationCheck, settleCoalescedFreshContinuationHolds } from "./fresh-continuation-fallback.js"; @@ -368,6 +369,7 @@ export async function deliverReconciledExecutions( db: Db, wake: ReturnType["wakeup"], ) { + if (isReattachPhaseActive()) return; const recoveryReasons = [...RECOVERY_WAKE_REASONS]; const sourceRun = alias(heartbeatRuns, "continuation_source_run"); const seenActionIds = new Set(); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index a61e9ea1da3..85387808006 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -46,7 +46,7 @@ import { dotRunnerBroker } from "./dot-runner-broker.js"; import { withRunExecutionTiming, measureRunPhase, measureWorkspaceSyncPhase, isWorkspaceSyncDetailPhase } from "./execution-preparation-timing.js"; import { createHeartbeatPreparationTiming } from "./heartbeat/preparation-timing.js"; import { createHeartbeatInstructionCopyRelease, createHeartbeatInstructionWorkingCopy } from "./heartbeat/instruction-working-copy.js"; -import { AgentDirectoryPreparationDeferral, deferRunForBusyAgentDirectory, prepareWithAgentDirectoryDeferral } from "./heartbeat/agent-directory-busy.js"; +import { AgentDirectoryPreparationDeferral, deferRunForBusyAgentDirectory, prepareWithAgentDirectoryDeferral, REATTACH_AGENT_DIRECTORY_WAIT_MS } from "./heartbeat/agent-directory-busy.js"; import { preparePredecessorRetirementFallback } from "./heartbeat/predecessor-retirement-fallback.js"; import { prepareHeartbeatRuntimeServiceIntents, startHeartbeatRuntimeServices, recordHeartbeatAdapterRuntimeServices } from "./heartbeat/runtime-services.js"; import { bindHeartbeatRunnerHomeMode, buildHeartbeatWorkspaceConfiguration, prepareHeartbeatExecutionConfiguration } from "./heartbeat/execution-configuration.js"; @@ -1109,6 +1109,7 @@ export function heartbeatService( reapOrphanedRuns, sweepOrphanedActiveLeases, drainRunningRunsForShutdown, + handOffNativeRunsForShutdown, registerScheduledRetryCancellationWork, registerRetainedNativeSessionCleanupWork, registerNativeWorkspaceFinalizationWork, @@ -3419,6 +3420,7 @@ export function heartbeatService( appendRunEvent, getCurrentUserRedactionOptions, assertWriterAuthority: createRunLogWriterAuthority(db, run, runOptions.nativeLeaseOwner), runtimeLogSecretValues: observabilitySecretValues(observability), + deferNativeLogRecovery: runOptions.nativeRestartRecovery?.kind === "reattach_remote_runner", signal: executionControl.controller.signal, }); const goalCheckpointSession: { @@ -3669,7 +3671,8 @@ export function heartbeatService( }, { measure: timePreparation, prepareInstructions: async nativeLifecyclePolicy => { - runtimeConfig = await prepareWithAgentDirectoryDeferral(() => preparedInstructions.prepare({ runtimeConfig, nativeRuntimeResolution, hasInstructionFilesystem, nativeLifecyclePolicy })); + runtimeConfig = await prepareWithAgentDirectoryDeferral(() => preparedInstructions.prepare({ runtimeConfig, nativeRuntimeResolution, hasInstructionFilesystem, nativeLifecyclePolicy }), + { waitMs: runOptions.nativeRestartRecovery ? REATTACH_AGENT_DIRECTORY_WAIT_MS : 0 }); return { runtimeConfig, instructionCopy: preparedInstructions.instructionCopy }; }, }); @@ -4682,6 +4685,7 @@ export function heartbeatService( // gate on suppression should prefer this over the env-only resolver. resolveSchedulingSuppression: getSchedulingSuppression, drainRunningRunsForShutdown, + handOffNativeRunsForShutdown, drainActiveRunExecutions, scheduleNativeSessionResumeDispatch, startTaskDrain, diff --git a/server/src/services/heartbeat/agent-directory-busy.test.ts b/server/src/services/heartbeat/agent-directory-busy.test.ts new file mode 100644 index 00000000000..3ee4828c8a6 --- /dev/null +++ b/server/src/services/heartbeat/agent-directory-busy.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from "vitest"; +import { AgentDirectoryOperationBusyError } from "../agent-directory-working-copies.js"; +import { AgentDirectoryPreparationDeferral, prepareWithAgentDirectoryDeferral, REATTACH_AGENT_DIRECTORY_WAIT_MS } from "./agent-directory-busy.js"; + +describe("agent directory preparation while another run of the agent holds it", () => { + it("defers an ordinary start at once, with no wait added to the run-start path", async () => { + const sleep = vi.fn(async () => undefined); + const prepare = vi.fn(async () => { throw new AgentDirectoryOperationBusyError(); }); + await expect(prepareWithAgentDirectoryDeferral(prepare, { sleep })).rejects.toBeInstanceOf(AgentDirectoryPreparationDeferral); + expect(prepare).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + }); + + it("lets a reattach wait its turn in process instead of a deferred full re-preparation", async () => { + const sleep = vi.fn(async (_ms: number) => undefined); + let attempts = 0; + const prepare = vi.fn(async () => { + attempts += 1; + if (attempts < 4) throw new AgentDirectoryOperationBusyError(); + return "prepared"; + }); + await expect(prepareWithAgentDirectoryDeferral(prepare, { waitMs: REATTACH_AGENT_DIRECTORY_WAIT_MS, sleep })).resolves.toBe("prepared"); + expect(sleep.mock.calls.map(([ms]) => ms)).toEqual([100, 200, 400]); + }); + + it("still defers a reattach once its bounded wait runs out", async () => { + vi.useFakeTimers(); + try { + vi.setSystemTime(0); + const sleep = vi.fn(async (ms: number) => { vi.setSystemTime(Date.now() + ms); }); + const prepare = vi.fn(async () => { throw new AgentDirectoryOperationBusyError(); }); + await expect(prepareWithAgentDirectoryDeferral(prepare, { waitMs: 5_000, sleep })).rejects.toBeInstanceOf(AgentDirectoryPreparationDeferral); + expect(Date.now()).toBeLessThanOrEqual(5_000); + } finally { + vi.useRealTimers(); + } + }); + + it("does not retry other failures", async () => { + const prepare = vi.fn(async () => { throw new Error("other"); }); + await expect(prepareWithAgentDirectoryDeferral(prepare, { waitMs: 30_000, sleep: async () => undefined })).rejects.toThrow("other"); + expect(prepare).toHaveBeenCalledTimes(1); + }); +}); diff --git a/server/src/services/heartbeat/agent-directory-busy.ts b/server/src/services/heartbeat/agent-directory-busy.ts index 67bf108af94..8aa1611de79 100644 --- a/server/src/services/heartbeat/agent-directory-busy.ts +++ b/server/src/services/heartbeat/agent-directory-busy.ts @@ -18,12 +18,22 @@ export class AgentDirectoryPreparationDeferral extends Error { } } -export async function prepareWithAgentDirectoryDeferral(prepare: () => Promise): Promise { - try { - return await prepare(); - } catch (error) { - if (error instanceof AgentDirectoryOperationBusyError) throw new AgentDirectoryPreparationDeferral(error); - throw error; +export const REATTACH_AGENT_DIRECTORY_WAIT_MS = 30_000; + +export async function prepareWithAgentDirectoryDeferral(prepare: () => Promise, options: { + waitMs?: number; + sleep?: (ms: number) => Promise; +} = {}): Promise { + const deadline = Date.now() + (options.waitMs ?? 0); + const sleep = options.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms))); + for (let delayMs = 100; ; delayMs = Math.min(delayMs * 2, 2_000)) { + try { + return await prepare(); + } catch (error) { + if (!(error instanceof AgentDirectoryOperationBusyError)) throw error; + if (Date.now() + delayMs > deadline) throw new AgentDirectoryPreparationDeferral(error); + await sleep(delayMs); + } } } diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index fd9afa56d83..33c62c65645 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -623,6 +623,28 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { expect(deps.executeRun).not.toHaveBeenCalled(); }); + it("returns the claim to the queue when a reattach phase begins while the claim is being checked", async () => { + const f = await fixture("queued"); + const [queued] = await db.update(heartbeatRuns).set({ responsibleUserId: "queue-test-user", + contextSnapshot: { issueId: f.issue.id, wakeReason: "issue_commented" } }) + .where(eq(heartbeatRuns.id, f.run.id)).returning(); + const [issueBefore] = await db.select().from(issues).where(eq(issues.id, f.issue.id)); + let reattachPhaseActive = false; + const deps = callbacks(db); + deps.getAgent.mockResolvedValue(f.agent); + deps.isRunAdmissionClosed = () => false; + deps.isReattachPhaseActive = () => reattachPhaseActive; + deps.budgets.getInvocationBlock.mockImplementation(async () => { reattachPhaseActive = true; return null; }); + expect(await createHeartbeatQueue(db, deps).claimQueuedRun(queued)).toBeNull(); + const [stored] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id)); + expect(stored).toMatchObject({ status: "queued", startedAt: null, finishedAt: null, errorCode: null, + responsibleUserId: queued.responsibleUserId, runnerProfileJson: queued.runnerProfileJson }); + expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0]).toMatchObject({ + executionRunId: issueBefore.executionRunId, executionLockedAt: issueBefore.executionLockedAt }); + expect(deps.setWakeupStatus).not.toHaveBeenCalled(); + expect(deps.executeRun).not.toHaveBeenCalled(); + }); + it("keeps an automatic continuation queued while its legacy predecessor still owns cleanup", async () => { const { successor, lease, queue, deps, ...f } = await pendingPredecessorFixture(); expect(await queue.claimQueuedRun(successor)).toBeNull(); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 622827d66c3..ed49232fc19 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -2,7 +2,7 @@ import { QUEUED_MESSAGES_DELIVERED_REASON } from "../../modules/wake-queue/index import { connectionIntentService } from "../connection-intents.js"; import { verifyBlockerStallWake } from "../recovery/blocker-stall-wakeups.js"; import { CONTEXT_COMPACTION_WAKE_REASON, ContextCompactionError, isContextCompactionRun, readContextCompactionRequest } from "../context-compaction.js"; -import { isProcessRunAdmissionClosed } from "./run-admission.js"; +import { isProcessRunAdmissionClosed, isReattachPhaseActive as isProcessReattachPhaseActive } from "./run-admission.js"; import { continueFreshInTransaction, hasPendingFreshContinuation, isFreshIsolatedContinuationWake, settleCoalescedFreshContinuationHolds } from "../fresh-continuation-fallback.js"; import { deliverContinuationAsRun, foldWaitingRetriesIntoContinuation, foldWaitingRetriesIntoRecordedContinuation, undeliveredContinuation, undeliveredContinuationContext } from "../fresh-continuation-retry-fold.js"; @@ -250,6 +250,7 @@ export interface HeartbeatQueueDependencies extends Pick(run: T | null, live: { has(id: string): boolean }) => T | null; getSchedulingSuppression: () => Promise<{ suppressed: boolean; reason: "worktree_instance" | "database_restore_in_progress" | "task_drain" | null }>; isRunAdmissionClosed?: () => boolean; + isReattachPhaseActive?: () => boolean; /** Before a dispatch pass, refresh admission inputs that have gone stale; null when current. */ refreshRunAdmission?: () => Promise | null; canStartNewExternalBotWork: (agent: typeof agents.$inferSelect) => Promise; @@ -471,7 +472,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende isHeartbeatRunTerminalStatus, liveRunExecutions, getSchedulingSuppression, - isRunAdmissionClosed = isProcessRunAdmissionClosed, + isRunAdmissionClosed: isStartAdmissionClosed = isProcessRunAdmissionClosed, + isReattachPhaseActive = isProcessReattachPhaseActive, refreshRunAdmission = refreshRunStartYieldIfStale, resumeExecutionWaitComments, getWorktreeExecutionCutoff, @@ -498,6 +500,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende filterZombieCoalesceTarget, } = dependencies; + const isRunAdmissionClosed = () => isStartAdmissionClosed() || isReattachPhaseActive(); + const { parseHeartbeatPolicy, getHeartbeatDailyCapBlock, hasActionableTimerWork, nextTimerHeartbeatAt } = createHeartbeatTimerPolicy(db); // claimQueuedRun works from the caller's copy of a queued run. Another @@ -2289,7 +2293,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // the full queued-run pass. Every run still passes the ordinary per-agent // claim and dispatch gates, exactly as a new wakeup during startup would. async function resumeFreshQueuedRunsAtStartup() { - if ((await getSchedulingSuppression()).suppressed) return { agents: 0 }; + if (isRunAdmissionClosed() || (await getSchedulingSuppression()).suppressed) return { agents: 0 }; const cutoff = await getWorktreeExecutionCutoff(); const rows = await db .selectDistinct({ agentId: heartbeatRuns.agentId }) diff --git a/server/src/services/heartbeat/reattach-phase.test.ts b/server/src/services/heartbeat/reattach-phase.test.ts new file mode 100644 index 00000000000..4a6fb49d195 --- /dev/null +++ b/server/src/services/heartbeat/reattach-phase.test.ts @@ -0,0 +1,616 @@ +import type { Db } from "@paperclipai/db"; +import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { createHeartbeatQueueTestDependencies as callbacks } from "../../__tests__/helpers/heartbeat-queue.js"; +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { createHeartbeatQueue } from "./queue.js"; +import { + closeProcessRunAdmission, + enableReattachPhase, + isReattachPhaseActive, + joinReattachPhase, + markReattachAuthenticated, + onReattachPhaseEnd, + pendingReattachRunIds, + reattachPhaseRecheckAt, + refineReattachClaimable, + resetProcessRunAdmissionForTests, + resetReattachPhaseForTests, + settleReattachRun, + type ClaimableReattachRun, + type ReattachPhaseSummary, +} from "./run-admission.js"; +import { + claimableReattachRun, + handOverMoment, + listLiveRunsOwnedByPreviousControllers, + noteReattachRunsClaimed, + REATTACH_PHASE_DEFAULT_CAP_MS, + reattachPhaseCapMs, + notifyReattachHandOver, + registerReattachPhaseWork, + settleFinishedReattachRuns, + stopReattachWatch, +} from "./reattach-phase.js"; +import { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; +import { isReattachPhaseActive as gateUsedBySweep } from "./run-admission.js"; + +const coordinators: ReturnType[] = []; + +async function startReattachPhase(db: Db, options: Parameters[2]) { + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: () => undefined }); + coordinators.push(coordinator); + await registerReattachPhaseWork(coordinator, db, options)?.ready; + return coordinator; +} + +afterEach(async () => { + await stopReattachWatch(); + for (const coordinator of coordinators.splice(0)) await coordinator.stop(); + resetReattachPhaseForTests(); + vi.useRealTimers(); +}); + +function guardedDatabase() { + const access = vi.fn(() => { throw new Error("Unexpected queue database access"); }); + return { db: new Proxy({}, { get: access }) as Db, access }; +} + +const handedOver = (runId: string, claimableAt: number): ClaimableReattachRun => ({ runId, reason: "handed_over", claimableAt }); + +describe("reattach phase signal", () => { + it("is false when no phase was opened, so a fresh process and every test start runs as before", () => { + expect(isReattachPhaseActive()).toBe(false); + }); + + it("is never true while nothing is claimable, even with the tracker enabled", () => { + enableReattachPhase(180_000, 1_000); + expect(isReattachPhaseActive(1_001)).toBe(false); + expect(joinReattachPhase([], {}, 1_005)).toEqual([]); + expect(isReattachPhaseActive(1_006)).toBe(false); + }); + + it("is never true when the tracker is disabled (cap 0, warm standby, scheduler off)", () => { + expect(joinReattachPhase([handedOver("run", 0)], {}, 0)).toEqual([]); + expect(isReattachPhaseActive(1)).toBe(false); + }); + + it("stays true until every claimable run is reached or given up, and reports latency from when each became claimable", () => { + const ended: ReattachPhaseSummary[] = []; + onReattachPhaseEnd(summary => ended.push(summary)); + enableReattachPhase(180_000, 0); + joinReattachPhase([handedOver("run-a", 50), handedOver("run-b", 50), handedOver("run-c", 50)], {}, 60); + settleReattachRun("run-a", "reached", 4_000); + expect(isReattachPhaseActive(4_001)).toBe(true); + settleReattachRun("run-unrelated", "reached", 4_500); + settleReattachRun("run-b", "given_up", 6_000); + settleReattachRun("run-a", "given_up", 6_500); + expect(isReattachPhaseActive(6_600)).toBe(true); + settleReattachRun("run-c", "reached", 9_000); + expect(isReattachPhaseActive(9_001)).toBe(false); + expect(ended).toHaveLength(1); + expect(ended[0]).toMatchObject({ reason: "all_settled", startReason: "handed_over", liveRuns: 3, reached: 2, givenUp: 1, + outstanding: 0, durationMs: 8_950 }); + expect(ended[0].runs).toEqual([ + { runId: "run-a", outcome: "reached", latencyMs: 3_950, authenticatedMs: null }, + { runId: "run-b", outcome: "given_up", latencyMs: 5_950, authenticatedMs: null }, + { runId: "run-c", outcome: "reached", latencyMs: 8_950, authenticatedMs: null }, + ]); + }); + + it("records authentication as a mark, not as reached, and counts runs reached after the 150 s failure threshold", () => { + const ended: ReattachPhaseSummary[] = []; + onReattachPhaseEnd(summary => ended.push(summary)); + enableReattachPhase(300_000, 0); + joinReattachPhase([handedOver("fast", 0), handedOver("slow", 0)], {}, 0); + markReattachAuthenticated("fast", 2_000); + markReattachAuthenticated("slow", 3_000); + expect(isReattachPhaseActive(3_001)).toBe(true); + settleReattachRun("fast", "reached", 20_000); + settleReattachRun("slow", "reached", 200_000); + expect(ended[0]).toMatchObject({ authenticated: 2, reached: 2, reachedAfterThreshold: 1, maxReachLatencyMs: 200_000 }); + expect(ended[0].runs.map(run => run.authenticatedMs)).toEqual([2_000, 3_000]); + }); + + it("moves the cap earlier when a claimed run turns out to have been handed over before its claim", () => { + enableReattachPhase(1_000, 0); + joinReattachPhase([{ runId: "claimed", reason: "owner_gone", claimableAt: 10_000 }], { announce: false }, 10_000); + expect(isReattachPhaseActive(10_900)).toBe(true); + refineReattachClaimable([handedOver("claimed", 9_200)], 10_100); + expect(isReattachPhaseActive(10_150)).toBe(true); + expect(isReattachPhaseActive(10_250)).toBe(false); + }); + + it("ends at the cap, counted from when the runs became claimable, so one wedged Runner cannot hold run starts", () => { + vi.useFakeTimers(); + vi.setSystemTime(0); + const ended: ReattachPhaseSummary[] = []; + onReattachPhaseEnd(summary => ended.push(summary)); + enableReattachPhase(180_000); + vi.advanceTimersByTime(500_000); + expect(isReattachPhaseActive()).toBe(false); + joinReattachPhase([handedOver("wedged", 500_000), handedOver("fine", 500_000)]); + settleReattachRun("fine", "reached", 502_000); + expect(isReattachPhaseActive(679_999)).toBe(true); + vi.advanceTimersByTime(180_000); + expect(ended).toEqual([expect.objectContaining({ reason: "cap_reached", liveRuns: 2, reached: 1, outstanding: 1, durationMs: 180_000 })]); + expect(isReattachPhaseActive()).toBe(false); + }); + + it("counts a lease that lapsed before this server started from the server's start, not from the lapse", () => { + const ended: ReattachPhaseSummary[] = []; + onReattachPhaseEnd(summary => ended.push(summary)); + enableReattachPhase(180_000, 600_000); + joinReattachPhase([{ runId: "orphan", reason: "lease_expired", claimableAt: 10_000 }], {}, 600_100); + expect(isReattachPhaseActive(600_200)).toBe(true); + settleReattachRun("orphan", "reached", 605_000); + expect(ended[0]).toMatchObject({ startReason: "lease_expired", maxReachLatencyMs: 5_000 }); + }); + + it("re-arms for a later batch with its own cap, and never takes back a run it already reached or gave up", () => { + const ended: ReattachPhaseSummary[] = []; + onReattachPhaseEnd(summary => ended.push(summary)); + enableReattachPhase(30_000, 0); + joinReattachPhase([handedOver("first", 1_000)], {}, 1_000); + expect(isReattachPhaseActive(30_999)).toBe(true); + expect(isReattachPhaseActive(31_000)).toBe(false); + expect(joinReattachPhase([handedOver("first", 40_000)], {}, 40_000)).toEqual([]); + expect(isReattachPhaseActive(40_001)).toBe(false); + expect(joinReattachPhase([handedOver("second", 40_000)], {}, 40_000)).toEqual(["second"]); + expect(isReattachPhaseActive(69_999)).toBe(true); + settleReattachRun("second", "reached", 41_000); + expect(ended.map(summary => [summary.reason, summary.runs.map(run => run.runId)])).toEqual([ + ["cap_reached", ["first"]], ["all_settled", ["second"]], + ]); + }); + + it("extends the cap for runs that join a phase in progress (a hand-over in two steps)", () => { + enableReattachPhase(30_000, 0); + joinReattachPhase([handedOver("early", 0)], {}, 0); + joinReattachPhase([handedOver("late", 20_000)], {}, 20_000); + settleReattachRun("early", "reached", 21_000); + expect(isReattachPhaseActive(45_000)).toBe(true); + expect(isReattachPhaseActive(50_000)).toBe(false); + }); + + it("asks deferred work to come back within the phase, never later than its cap", () => { + enableReattachPhase(30_000, 0); + joinReattachPhase([handedOver("run", 0)], {}, 0); + expect(reattachPhaseRecheckAt(1_000).getTime()).toBe(11_000); + expect(reattachPhaseRecheckAt(25_000).getTime()).toBe(30_000); + }); + + it("reads its cap from the environment within bounds", () => { + expect(reattachPhaseCapMs({})).toBe(REATTACH_PHASE_DEFAULT_CAP_MS); + expect(reattachPhaseCapMs({ PAPERCLIP_REATTACH_PHASE_CAP_MS: "240000" })).toBe(240_000); + expect(reattachPhaseCapMs({ PAPERCLIP_REATTACH_PHASE_CAP_MS: "10" })).toBe(5_000); + expect(reattachPhaseCapMs({ PAPERCLIP_REATTACH_PHASE_CAP_MS: "99999999" })).toBe(600_000); + expect(reattachPhaseCapMs({ PAPERCLIP_REATTACH_PHASE_CAP_MS: "0" })).toBe(0); + expect(reattachPhaseCapMs({ PAPERCLIP_REATTACH_PHASE_CAP_MS: "soon" })).toBe(REATTACH_PHASE_DEFAULT_CAP_MS); + }); + + it("is the same gate the assigned-work sweep and reconciled continuation delivery read", () => { + enableReattachPhase(180_000); + joinReattachPhase([handedOver("run", Date.now())]); + expect(gateUsedBySweep()).toBe(true); + }); + + it("reads a run as claimable only once its lease was released or has expired, and says why", () => { + const now = 100_000; + expect(claimableReattachRun({ runId: "live", leaseExpiresAt: new Date(now + 1), recoveryState: null }, now)).toBeNull(); + expect(claimableReattachRun({ runId: "live", leaseExpiresAt: new Date(now + 1), recoveryState: "awaiting_runner_reattach" }, now)).toBeNull(); + expect(claimableReattachRun({ runId: "h", leaseExpiresAt: new Date(now - 5), recoveryState: "awaiting_runner_reattach" }, now)) + .toEqual({ runId: "h", reason: "handed_over", claimableAt: now - 5 }); + expect(claimableReattachRun({ runId: "e", leaseExpiresAt: new Date(now - 7), recoveryState: null }, now)) + .toEqual({ runId: "e", reason: "lease_expired", claimableAt: now - 7 }); + expect(claimableReattachRun({ runId: "g", leaseExpiresAt: null, recoveryState: null }, now)) + .toEqual({ runId: "g", reason: "owner_gone", claimableAt: now }); + }); + + it("finds the previous owner's hand-over behind this server's own recovery entries", () => { + const history = [ + { at: "2026-11-13T10:00:00.000Z", disposition: "reattach_remote_runner", controllerBootId: "older" }, + { at: "2026-11-13T10:05:00.000Z", disposition: "restart_suspended" }, + { at: "2026-11-13T10:05:00.020Z", disposition: "reattach_remote_runner", controllerBootId: "mine" }, + ]; + expect(handOverMoment(history, "mine")).toBe(Date.parse("2026-11-13T10:05:00.000Z")); + expect(handOverMoment([history[0], history[2]], "mine")).toBeNull(); + expect(handOverMoment([], "mine")).toBeNull(); + }); + + it("ignores claims while the tracker is not running", () => { + noteReattachRunsClaimed(["run"]); + expect(isReattachPhaseActive()).toBe(false); + }); +}); + +describe("queued run starts during the reattach phase", () => { + it("claimQueuedRun leaves a queued run queued and unstarted, in the admission gate's shape", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.isRunAdmissionClosed = () => false; + const queue = createHeartbeatQueue(database.db, deps); + enableReattachPhase(180_000); + joinReattachPhase([handedOver("live-run", Date.now())]); + const queued = { id: "run-1", status: "queued" } as Parameters[0]; + expect(await queue.claimQueuedRun(queued)).toBeNull(); + await queue.startNextQueuedRunForAgent("agent"); + await queue.resumeQueuedRuns(); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.getAgent).not.toHaveBeenCalled(); + expect(deps.executeRun).not.toHaveBeenCalled(); + }); + + it("does not hold a run that is already running (the reattach itself)", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + const queue = createHeartbeatQueue(database.db, deps); + enableReattachPhase(180_000); + joinReattachPhase([handedOver("live-run", Date.now())]); + const running = { id: "live-run", status: "running" } as Parameters[0]; + expect(await queue.claimQueuedRun(running)).toBe(running); + }); +}); + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_QUEUE_TEST_DATABASE_URL"); +const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); +if (!support.supported) console.warn(`Skipping reattach phase database tests: ${support.reason}`); +describe.skipIf(!support.supported)("reattach phase discovery", () => { + let database: Awaited> | undefined; + let db: Db; + beforeAll(async () => { + if (!external) database = await startEmbeddedPostgresTestDatabase("paperclip-reattach-phase"); + db = createDb(external ?? database!.connectionString, { maxConnections: 4 }); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { + await db?.$client.end(); + await database?.cleanup(); + }); + + async function seed(prefix: string) { + const [company] = await db.insert(companies).values({ name: `Reattach ${prefix}`, issuePrefix: prefix }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Runner agent", status: "running", + adapterType: "paperclip_runner", adapterConfig: {} }).returning(); + const run = async (status: "running" | "queued" | "succeeded", finalization?: Partial) => { + const [issue] = await db.insert(issues).values({ companyId: company.id, title: `Task ${status}`, status: "in_progress", + assigneeAgentId: agent.id }).returning(); + const [row] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status, + runtimeMode: "native", nativeIssueId: issue.id, invocationSource: "assignment", contextSnapshot: { issueId: issue.id } }).returning(); + if (finalization) await db.insert(nativeRunFinalizations).values({ runId: row.id, companyId: company.id, issueId: issue.id, + phase: "observed", ...finalization }); + return row.id; + }; + return { run }; + } + + it("lists only live runs another controller owns, and nothing on a fresh or own-only instance", async () => { + const { run } = await seed("RPA"); + const previous = await run("running", { leaseOwner: "native:old", controllerBootId: "old-boot" }); + const legacy = await run("running", { leaseOwner: "native:legacy", controllerBootId: null }); + await run("running", { leaseOwner: "native:mine", controllerBootId: "this-boot" }); + await run("running", { leaseOwner: "native-finalizer:x", controllerBootId: "old-boot" }); + await run("running", { leaseOwner: null, controllerBootId: "old-boot" }); + await run("running", { leaseOwner: "native:old", controllerBootId: "old-boot", phase: "committed" }); + await run("succeeded", { leaseOwner: "native:old", controllerBootId: "old-boot" }); + await run("queued"); + expect((await listLiveRunsOwnedByPreviousControllers(db, "this-boot")).sort()).toEqual([previous, legacy].sort()); + }); + + it("settles runs that finished by any path as given up and keeps live ones pending", async () => { + const { run } = await seed("RPB"); + const live = await run("running", { leaseOwner: "native:old", controllerBootId: "old-boot-b" }); + const finished = await run("running", { leaseOwner: "native:old", controllerBootId: "old-boot-b" }); + enableReattachPhase(180_000); + joinReattachPhase([handedOver(live, Date.now()), handedOver(finished, Date.now())]); + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, finished)); + expect(await settleFinishedReattachRuns(db)).toBe(1); + expect(isReattachPhaseActive()).toBe(true); + settleReattachRun(live, "reached"); + expect(isReattachPhaseActive()).toBe(false); + }); + + it("starts and ends the phase from the database, calling back once it ends", async () => { + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.status, "running")); + const ended: ReattachPhaseSummary[] = []; + await startReattachPhase(db, { capMs: 60_000, bootId: "boot-c", onEnd: summary => ended.push(summary) }); + expect(isReattachPhaseActive()).toBe(false); + expect(ended).toEqual([expect.objectContaining({ reason: "nothing_to_reattach", gatedStarts: false })]); + const { run } = await seed("RPC"); + const only = await run("running", { leaseOwner: "native:old", controllerBootId: "old-boot-c" }); + await startReattachPhase(db, { capMs: 60_000, bootId: "boot-c", recheckMs: 50, onEnd: summary => ended.push(summary) }); + expect(isReattachPhaseActive()).toBe(true); + settleReattachRun(only, "reached"); + expect(isReattachPhaseActive()).toBe(false); + expect(ended.at(-1)).toMatchObject({ reason: "all_settled", liveRuns: 1, reached: 1, gatedStarts: true }); + }); + + describe("timing across a release swap", () => { + const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + const liveLease = () => ({ leaseOwner: "native-execution:outgoing", controllerBootId: "outgoing-boot", + leaseExpiresAt: new Date(Date.now() + 60_000) }); + + async function retireEarlierRuns() { + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.status, "running")); + } + + async function handOver(runIds: string[], at = new Date(), options: { awaitJoin?: boolean } = {}) { + for (const runId of runIds) { + await db.transaction(async tx => { + await notifyReattachHandOver(tx, runId); + await tx.update(nativeRunFinalizations).set({ leaseExpiresAt: at, recoveryState: "awaiting_runner_reattach", + recoveryHistory: [{ at: at.toISOString(), restartKind: "graceful", disposition: "restart_suspended", reason: "SIGTERM" }] }) + .where(eq(nativeRunFinalizations.runId, runId)); + }); + } + if (options.awaitJoin === false) return; + await vi.waitFor(() => expect(runIds.filter(runId => !pendingReattachRunIds().includes(runId))).toEqual([]), + { timeout: 2_000, interval: 10 }); + } + + async function queuedClaimPassesGate() { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.isRunAdmissionClosed = () => false; + const queue = createHeartbeatQueue(database.db, deps); + const queued = { id: "queued-run", status: "queued" } as Parameters[0]; + const claimed = await queue.claimQueuedRun(queued).catch((error: Error) => error); + return database.access.mock.calls.length > 0 || claimed instanceof Error; + } + + it("does not hold queued starts while the previous owner is alive and not handing over", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPD"); + await run("running", liveLease()); + await run("running", liveLease()); + const ended: ReattachPhaseSummary[] = []; + await startReattachPhase(db, { capMs: 200, bootId: "incoming-boot", recheckMs: 20, onEnd: summary => ended.push(summary) }); + expect(isReattachPhaseActive()).toBe(false); + expect(await queuedClaimPassesGate()).toBe(true); + await sleep(350); + expect(isReattachPhaseActive()).toBe(false); + expect(ended.filter(summary => summary.reason === "cap_reached")).toEqual([]); + }); + + it("closes the gate when the previous owner hands its runs over, and counts reach latency from the hand-over", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPE"); + const first = await run("running", liveLease()); + const second = await run("running", liveLease()); + const ended: ReattachPhaseSummary[] = []; + await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20, onEnd: summary => ended.push(summary) }); + await sleep(400); + expect(isReattachPhaseActive()).toBe(false); + await handOver([first, second]); + await vi.waitFor(() => expect(isReattachPhaseActive()).toBe(true), { timeout: 2_000, interval: 10 }); + expect(await queuedClaimPassesGate()).toBe(false); + settleReattachRun(first, "reached"); + expect(isReattachPhaseActive()).toBe(true); + settleReattachRun(second, "reached"); + expect(isReattachPhaseActive()).toBe(false); + const phases = ended.filter(summary => summary.reason !== "nothing_to_reattach"); + expect(phases).toEqual([expect.objectContaining({ reason: "all_settled", liveRuns: 2, reached: 2, gatedStarts: true })]); + expect(phases[0].maxReachLatencyMs).toBeLessThan(350); + expect(phases[0].durationMs).toBeLessThan(350); + }); + + it("runs the cap from the hand-over, so a wedged run ends the phase without the overlap eating it", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPF"); + const wedged = await run("running", liveLease()); + const fine = await run("running", liveLease()); + const ended: ReattachPhaseSummary[] = []; + await startReattachPhase(db, { capMs: 300, bootId: "incoming-boot", recheckMs: 20, onEnd: summary => ended.push(summary) }); + await sleep(450); + expect(ended.filter(summary => summary.reason === "cap_reached")).toEqual([]); + await handOver([wedged, fine]); + await vi.waitFor(() => expect(isReattachPhaseActive()).toBe(true), { timeout: 2_000, interval: 10 }); + settleReattachRun(fine, "reached"); + await vi.waitFor(() => expect(ended.some(summary => summary.reason === "cap_reached")).toBe(true), { timeout: 2_000, interval: 10 }); + const capped = ended.find(summary => summary.reason === "cap_reached")!; + expect(capped).toMatchObject({ liveRuns: 2, reached: 1, outstanding: 1, gatedStarts: true }); + expect(capped.durationMs).toBeGreaterThanOrEqual(250); + expect(capped.durationMs).toBeLessThan(450); + expect(isReattachPhaseActive()).toBe(false); + }); + + it("begins at a killed owner's lease deadline, without any hand-over or notification", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPG"); + const leaseEndsAt = Date.now() + 400; + const orphan = await run("running", { ...liveLease(), leaseExpiresAt: new Date(leaseEndsAt) }); + const ended: ReattachPhaseSummary[] = []; + const coordinator = await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20, onEnd: summary => ended.push(summary) }); + expect(coordinator.nextWakeAt()).toBe(leaseEndsAt); + await sleep(100); + expect(isReattachPhaseActive()).toBe(false); + await vi.waitFor(() => expect(isReattachPhaseActive()).toBe(true), { timeout: 2_000, interval: 10 }); + expect(Date.now()).toBeGreaterThanOrEqual(leaseEndsAt); + settleReattachRun(orphan, "reached"); + expect(ended.filter(summary => summary.reason !== "nothing_to_reattach")) + .toEqual([expect.objectContaining({ reason: "all_settled", startReason: "lease_expired", liveRuns: 1, reached: 1 })]); + }); + + it("re-arms for a later batch, including runs the previous owner started after discovery", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPH"); + const early = await run("running", liveLease()); + const ended: ReattachPhaseSummary[] = []; + let resumes = 0; + await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20, + onEnd: summary => { ended.push(summary); if (summary.gatedStarts) resumes += 1; } }); + const late = await run("running", liveLease()); + await handOver([early]); + await vi.waitFor(() => expect(isReattachPhaseActive()).toBe(true), { timeout: 2_000, interval: 10 }); + settleReattachRun(early, "reached"); + expect(isReattachPhaseActive()).toBe(false); + expect(resumes).toBe(1); + await sleep(100); + expect(isReattachPhaseActive()).toBe(false); + await handOver([late]); + await vi.waitFor(() => expect(isReattachPhaseActive()).toBe(true), { timeout: 2_000, interval: 10 }); + settleReattachRun(late, "reached"); + expect(isReattachPhaseActive()).toBe(false); + expect(resumes).toBe(2); + expect(ended.filter(summary => summary.reason === "all_settled").map(summary => summary.runs.map(entry => entry.runId))) + .toEqual([[early], [late]]); + }); + + it("closes the gate at once when this server claims a run between two lease reads, dated from the hand-over", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPJ"); + const claimed = await run("running", liveLease()); + const ended: ReattachPhaseSummary[] = []; + await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 60_000, onEnd: summary => ended.push(summary) }); + expect(isReattachPhaseActive()).toBe(false); + await sleep(60); + const handedOverAt = new Date(Date.now() - 30); + await db.update(nativeRunFinalizations).set({ controllerBootId: "incoming-boot", leaseExpiresAt: new Date(Date.now() + 60_000), + recoveryHistory: [ + { at: handedOverAt.toISOString(), restartKind: "graceful", disposition: "restart_suspended", reason: "SIGTERM" }, + { at: new Date().toISOString(), restartKind: "hard", disposition: "reattach_remote_runner", controllerBootId: "incoming-boot" }, + ] }).where(eq(nativeRunFinalizations.runId, claimed)); + noteReattachRunsClaimed([claimed]); + expect(isReattachPhaseActive()).toBe(true); + await sleep(100); + settleReattachRun(claimed, "reached"); + const phase = ended.find(summary => summary.reason === "all_settled")!; + expect(phase).toMatchObject({ startReason: "handed_over", liveRuns: 1, reached: 1 }); + expect(phase.maxReachLatencyMs).toBeGreaterThanOrEqual(125); + }); + + it("retries a failed first listing instead of going quiet, and still finds the previous owner's runs", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPM"); + const lease = liveLease(); + const live = await run("running", lease); + const select = vi.spyOn(db, "select"); + select.mockImplementationOnce(() => { throw new Error("transient database error"); }); + const coordinator = await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20 }); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(lease.leaseExpiresAt.getTime()), { timeout: 2_000, interval: 10 }); + await handOver([live]); + expect(isReattachPhaseActive()).toBe(true); + select.mockRestore(); + settleReattachRun(live, "reached"); + }); + + it("keeps a hand-over notification whose listing failed until a listing succeeds", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPN"); + const ended: ReattachPhaseSummary[] = []; + await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20, onEnd: summary => ended.push(summary) }); + expect(ended).toEqual([expect.objectContaining({ reason: "nothing_to_reattach" })]); + const live = await run("running", liveLease()); + const select = vi.spyOn(db, "select"); + select.mockImplementationOnce(() => { throw new Error("transient database error"); }); + await handOver([live]); + expect(isReattachPhaseActive()).toBe(true); + select.mockRestore(); + settleReattachRun(live, "reached"); + }); + + it("starts the claim of handed-over runs within a second of the hand-over notification", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPO"); + const first = await run("running", liveLease()); + const second = await run("running", liveLease()); + const claimed: Array<{ runIds: string[]; at: number }> = []; + await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20, + claim: async runIds => { claimed.push({ runIds, at: Date.now() }); } }); + await sleep(100); + expect(claimed).toEqual([]); + const handedOverAt = Date.now(); + await handOver([first, second]); + expect(claimed.flatMap(entry => entry.runIds).sort()).toEqual([first, second].sort()); + expect(Math.max(...claimed.map(entry => entry.at)) - handedOverAt).toBeLessThan(1_000); + settleReattachRun(first, "reached"); + settleReattachRun(second, "reached"); + }); + + it("keeps an in-flight claim inside the worker's lifetime and starts none once shutdown closed admission", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPP"); + const first = await run("running", liveLease()); + const second = await run("running", liveLease()); + let finishClaim!: () => void; + const claims: string[][] = []; + const coordinator = await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20, + claim: runIds => { claims.push(runIds); return new Promise(resolve => { finishClaim = resolve; }); } }); + await handOver([first]); + expect(claims).toEqual([[first]]); + let stopped = false; + const stopping = coordinator.stop().then(() => { stopped = true; }); + await sleep(100); + expect(stopped).toBe(false); + finishClaim(); + await stopping; + expect(stopped).toBe(true); + closeProcessRunAdmission(); + try { + await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20, + claim: async runIds => { claims.push(runIds); } }); + await handOver([second]); + expect(claims).toEqual([[first]]); + } finally { + resetProcessRunAdmissionForTests(); + settleReattachRun(first, "reached"); + settleReattachRun(second, "reached"); + } + }); + + it("does no database work and keeps no deadline on an instance with nothing to reattach", async () => { + await retireEarlierRuns(); + const select = vi.spyOn(db, "select"); + const coordinator = await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20 }); + const startupQueries = select.mock.calls.length; + expect(coordinator.nextWakeAt()).toBeNull(); + await sleep(200); + expect(select.mock.calls.length).toBe(startupQueries); + select.mockRestore(); + }); + + it("waits for the hand-over notification or the lease deadline instead of polling while the previous owner drives its runs", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPK"); + const lease = liveLease(); + const live = await run("running", lease); + const select = vi.spyOn(db, "select"); + const coordinator = await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20 }); + const startupQueries = select.mock.calls.length; + expect(coordinator.nextWakeAt()).toBe(lease.leaseExpiresAt.getTime()); + await sleep(200); + expect(select.mock.calls.length).toBe(startupQueries); + select.mockRestore(); + await handOver([live]); + await vi.waitFor(() => expect(isReattachPhaseActive()).toBe(true), { timeout: 2_000, interval: 10 }); + settleReattachRun(live, "reached"); + expect(isReattachPhaseActive()).toBe(false); + }); + + it("stops with the delivery coordinator at shutdown and ignores a later hand-over", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPL"); + const live = await run("running", liveLease()); + const coordinator = await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20 }); + await coordinator.stop(); + expect(coordinator.nextWakeAt()).toBeNull(); + await handOver([live], new Date(), { awaitJoin: false }); + await sleep(200); + expect(isReattachPhaseActive()).toBe(false); + expect(pendingReattachRunIds()).toEqual([]); + }); + + it("is never true on an instance whose live runs are all its own", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPI"); + await run("running", { leaseOwner: "native-execution:mine", controllerBootId: "incoming-boot", leaseExpiresAt: new Date(Date.now() - 1) }); + const ended: ReattachPhaseSummary[] = []; + await startReattachPhase(db, { capMs: 60_000, bootId: "incoming-boot", recheckMs: 20, onEnd: summary => ended.push(summary) }); + expect(isReattachPhaseActive()).toBe(false); + await sleep(100); + expect(isReattachPhaseActive()).toBe(false); + expect(ended).toEqual([expect.objectContaining({ reason: "nothing_to_reattach", gatedStarts: false })]); + }); + }); +}); diff --git a/server/src/services/heartbeat/reattach-phase.ts b/server/src/services/heartbeat/reattach-phase.ts new file mode 100644 index 00000000000..50a03d29b3d --- /dev/null +++ b/server/src/services/heartbeat/reattach-phase.ts @@ -0,0 +1,244 @@ +import { and, eq, inArray, isNotNull, isNull, ne, notLike, or } from "drizzle-orm"; +import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { logger } from "../../middleware/logger.js"; +import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; +import { currentControllerBootId } from "../controller-process-identity.js"; +import { + announceReattachPhase, + enableReattachPhase, + hasJoinedReattachPhase, + isProcessRunAdmissionClosed, + isReattachPhaseEnabled, + joinReattachPhase, + onReattachPhaseEnd, + pendingReattachRunIds, + refineReattachClaimable, + reportNothingToReattach, + settleReattachRun, + type ClaimableReattachRun, + type ReattachClaimableReason, +} from "./run-admission.js"; + +export const REATTACH_PHASE_DEFAULT_CAP_MS = 180_000; +export const REATTACH_PHASE_MIN_CAP_MS = 5_000; +export const REATTACH_PHASE_MAX_CAP_MS = 600_000; +export const REATTACH_PHASE_RECHECK_MS = 2_000; + +export function reattachPhaseCapMs(env: NodeJS.ProcessEnv = process.env): number { + const raw = env.PAPERCLIP_REATTACH_PHASE_CAP_MS?.trim(); + if (!raw) return REATTACH_PHASE_DEFAULT_CAP_MS; + const parsed = Number(raw); + if (!Number.isFinite(parsed)) return REATTACH_PHASE_DEFAULT_CAP_MS; + if (parsed <= 0) return 0; + return Math.min(REATTACH_PHASE_MAX_CAP_MS, Math.max(REATTACH_PHASE_MIN_CAP_MS, Math.round(parsed))); +} + +export type PreviousControllerRunLease = { runId: string; leaseExpiresAt: Date | null; recoveryState: string | null }; + +export async function listPreviousControllerRunLeases(db: Db, bootId = currentControllerBootId()): Promise { + const rows = await db.select({ runId: heartbeatRuns.id, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt, + recoveryState: nativeRunFinalizations.recoveryState }) + .from(nativeRunFinalizations) + .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, nativeRunFinalizations.runId), + eq(heartbeatRuns.companyId, nativeRunFinalizations.companyId))) + .where(and( + eq(heartbeatRuns.status, "running"), + eq(heartbeatRuns.runtimeMode, "native"), + eq(nativeRunFinalizations.phase, "observed"), + isNull(nativeRunFinalizations.resultId), + isNotNull(nativeRunFinalizations.leaseOwner), + notLike(nativeRunFinalizations.leaseOwner, "native-finalizer:%"), + notLike(nativeRunFinalizations.leaseOwner, "native-cleanup:%"), + or(isNull(nativeRunFinalizations.controllerBootId), ne(nativeRunFinalizations.controllerBootId, bootId)), + )); + return [...new Map(rows.map(row => [row.runId, row])).values()]; +} + +export async function listLiveRunsOwnedByPreviousControllers(db: Db, bootId = currentControllerBootId()): Promise { + return (await listPreviousControllerRunLeases(db, bootId)).map(row => row.runId); +} + +export function claimableReattachRun(row: PreviousControllerRunLease, now = Date.now()): ClaimableReattachRun | null { + const expiresAt = row.leaseExpiresAt?.getTime() ?? null; + if (expiresAt !== null && expiresAt > now) return null; + const reason: ReattachClaimableReason = row.recoveryState === "awaiting_runner_reattach" ? "handed_over" + : expiresAt === null ? "owner_gone" : "lease_expired"; + return { runId: row.runId, reason, claimableAt: expiresAt ?? now }; +} + +export async function settleFinishedReattachRuns(db: Db): Promise { + const pending = pendingReattachRunIds(); + if (pending.length === 0) return 0; + const rows = await db.select({ runId: heartbeatRuns.id, status: heartbeatRuns.status, + resultId: nativeRunFinalizations.resultId, phase: nativeRunFinalizations.phase }) + .from(heartbeatRuns) + .leftJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.runId, heartbeatRuns.id), + eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId))) + .where(inArray(heartbeatRuns.id, pending)); + const live = new Set(rows.filter(row => row.status === "running" && row.resultId === null && row.phase === "observed") + .map(row => row.runId)); + const finished = pending.filter(runId => !live.has(runId)); + for (const runId of finished) settleReattachRun(runId, "given_up"); + return finished.length; +} + +type ReattachWorker = { wake: () => void; stop: () => Promise; ready: Promise }; + +type ReattachWatch = { + db: Db; + bootId: string; + initialized: boolean; + notified: boolean; + watching: boolean; + leases: Map; + worker: ReattachWorker | null; + stopListening: () => void; + claim?: (runIds: string[]) => Promise; +}; + +let watch: ReattachWatch | null = null; + +export async function notifyReattachHandOver(transaction: object, runId: string): Promise { + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.reattachPhase, runId); +} + +function observeLeases(current: ReattachWatch, rows: PreviousControllerRunLease[], now = Date.now()): string[] { + current.leases = new Map(rows.map(row => [row.runId, row])); + const unjoined = rows.filter(row => !hasJoinedReattachPhase(row.runId)); + const claimable = unjoined.map(row => claimableReattachRun(row, now)).filter((run): run is ClaimableReattachRun => run !== null); + const joined = joinReattachPhase(claimable, {}, now); + current.watching = unjoined.length > claimable.length; + return joined; +} + +async function claimJoinedRuns(current: ReattachWatch, joined: string[]): Promise { + if (joined.length === 0 || !current.claim || watch !== current || isProcessRunAdmissionClosed()) return; + await current.claim(joined) + .catch(error => logger.warn({ err: error, runIds: joined }, "reattach phase could not start the claim of handed-over runs")); +} + +export function nextReattachLeaseDeadline(leases: Iterable, now = Date.now()): number | null { + let next: number | null = null; + for (const row of leases) { + if (hasJoinedReattachPhase(row.runId)) continue; + const at = row.leaseExpiresAt?.getTime(); + if (at === undefined || at <= now) continue; + if (next === null || at < next) next = at; + } + return next; +} + +async function reconcileReattachWatch(current: ReattachWatch): Promise { + if (watch !== current) return; + if (!current.initialized) { + current.notified = false; + let rows: PreviousControllerRunLease[]; + try { + rows = await listPreviousControllerRunLeases(current.db, current.bootId); + } catch (error) { + current.notified = true; + throw error; + } + if (watch !== current) return; + current.initialized = true; + if (rows.length === 0) reportNothingToReattach(); + else { + await claimJoinedRuns(current, observeLeases(current, rows)); + logger.info({ liveRuns: rows.length, claimableNow: rows.filter(row => hasJoinedReattachPhase(row.runId)).length }, + "reattach phase watching live runs of a previous server: new run starts are held only once they become claimable"); + } + } else if (current.watching || current.notified) { + current.notified = false; + let rows: PreviousControllerRunLease[]; + try { + rows = await listPreviousControllerRunLeases(current.db, current.bootId); + } catch (error) { + current.notified = true; + throw error; + } + if (watch !== current) return; + await claimJoinedRuns(current, observeLeases(current, rows)); + } + if (pendingReattachRunIds().length > 0) await settleFinishedReattachRuns(current.db); +} + +export function noteReattachRunsClaimed(runIds: readonly string[], now = Date.now()): void { + const current = watch; + if (!current || !isReattachPhaseEnabled()) return; + const joined = joinReattachPhase(runIds.map(runId => { + const observed = current.leases.get(runId); + const claimable = observed ? claimableReattachRun(observed, now) : null; + return claimable ?? { runId, reason: "owner_gone" as const, claimableAt: now }; + }), { announce: false }, now); + if (joined.length === 0) return; + current.worker?.wake(); + void classifyClaimedRuns(current, joined, now) + .catch(error => logger.warn({ err: error }, "reattach phase could not read how claimed runs were released")) + .finally(() => announceReattachPhase(joined)); +} + +type RecoveryHistoryEntry = { at?: unknown; disposition?: unknown; controllerBootId?: unknown }; + +export function handOverMoment(history: readonly RecoveryHistoryEntry[], bootId: string): number | null { + for (let index = history.length - 1; index >= 0; index -= 1) { + const entry = history[index]; + if (entry?.controllerBootId === bootId) continue; + if (entry?.disposition !== "restart_suspended" || typeof entry.at !== "string") return null; + const at = Date.parse(entry.at); + return Number.isNaN(at) ? null : at; + } + return null; +} + +async function classifyClaimedRuns(current: ReattachWatch, runIds: string[], now: number): Promise { + const rows = await current.db.select({ runId: nativeRunFinalizations.runId, history: nativeRunFinalizations.recoveryHistory }) + .from(nativeRunFinalizations).where(inArray(nativeRunFinalizations.runId, runIds)); + refineReattachClaimable(rows.flatMap(row => { + const at = handOverMoment(Array.isArray(row.history) ? row.history as RecoveryHistoryEntry[] : [], current.bootId); + return at === null ? [] : [{ runId: row.runId, reason: "handed_over" as const, claimableAt: at }]; + }), now); +} + +export function registerReattachPhaseWork( + coordinator: Pick, "register">, + db: Db, + options: { + capMs?: number; + recheckMs?: number; + bootId?: string; + onEnd?: Parameters[0]; + claim?: (runIds: string[]) => Promise; + } = {}, +): ReattachWorker | null { + const capMs = options.capMs ?? reattachPhaseCapMs(); + if (capMs <= 0) return null; + if (watch) void stopReattachWatch(); + enableReattachPhase(capMs); + const current: ReattachWatch = { db, bootId: options.bootId ?? currentControllerBootId(), initialized: false, + notified: false, watching: false, leases: new Map(), worker: null, claim: options.claim, + stopListening: options.onEnd ? onReattachPhaseEnd(options.onEnd) : () => undefined }; + watch = current; + const release = () => { + if (watch !== current) return; + watch = null; + current.stopListening(); + }; + current.worker = coordinator.register(DELIVERY_QUEUES.reattachPhase, { + retryMs: options.recheckMs ?? REATTACH_PHASE_RECHECK_MS, + run: () => reconcileReattachWatch(current), + hasPending: async () => watch === current && pendingReattachRunIds().length > 0, + nextRunAt: () => watch === current && current.watching ? nextReattachLeaseDeadline(current.leases.values()) : null, + onSettled: () => { current.notified = true; }, + onStop: release, + }); + return current.worker; +} + +export async function stopReattachWatch(): Promise { + const current = watch; + if (!current) return; + watch = null; + current.stopListening(); + await current.worker?.stop(); +} diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index 86f751957bd..191f1ef963b 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -7,6 +7,7 @@ import { legacyWorkspaceRecoveryNotMigrated } from "../legacy-workspace-recovery import { pendingFailedNativeRetryCancellationCondition } from "../native-runtime/native-cancellation-request.js"; import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; import { registerLiveRecoveryWork } from "./live-recovery-work.js"; +import { noteReattachRunsClaimed, notifyReattachHandOver } from "./reattach-phase.js"; import { NATIVE_RETAINED_OWNER_WAIT } from "../native-runtime/native-controller-recovery.js"; import { nativeEnvironmentSettlementDisposition, nativeEnvironmentSettlementWork } from "../native-runtime/native-environment-settlement-work.js"; import { nativeWorkspaceFinalizationWork } from "../native-runtime/native-workspace-finalization-work.js"; @@ -376,6 +377,11 @@ function readNonEmptyString(value: unknown): string | null { /** Restart recovery shares the service lifecycle callbacks and execution ownership. */ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryDependencies) { + const runsHandedOffByThisController = new Set(); + const markHandedOffByThisController = (runId: string) => { + runsHandedOffByThisController.add(runId); + markRunHandedOffForShutdown(runId); + }; const { enterShutdown, scheduleBoundedRetryForRun, @@ -608,7 +614,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD isNativeSessionId(run.nativeSessionId), ) .map(({ run }) => run.id); - for (const runId of nativeRunIds) markRunHandedOffForShutdown(runId); + for (const runId of nativeRunIds) markHandedOffByThisController(runId); const detachedNativeSessions = await detachNativeSessionsForRestart(nativeRunIds); @@ -1020,6 +1026,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD db, liveRunExecutions: activeRunExecutions, handedOffForShutdown: isRunHandedOffForShutdown, + handedOffByThisController: runId => runsHandedOffByThisController.has(runId), runIds, restartKind, recoveryRequestId: intent?.recoveryRequestId ?? null, @@ -1042,6 +1049,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD disposition.kind === "resume_dead_runner" || disposition.kind === "bootstrap_incomplete", ); + if (options?.trigger !== "same_run_retry") noteReattachRunsClaimed(claims.map(claim => claim.runId)); for (const disposition of dispositions) { if ((disposition.kind === "awaiting_evidence" || disposition.kind === "blocked") && disposition.retryAt) { scheduleNativeSessionResumeDispatch(disposition.runId, disposition.retryAt, "restart"); @@ -1150,6 +1158,113 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD } } + const nativeRunsHandedOffEarly = new Set(); + + async function suspendNativeRunForShutdown(run: typeof heartbeatRuns.$inferSelect, signal: "SIGINT" | "SIGTERM", + now: Date, controllerBootId: string) { + const [owner] = await db.select({ bootId: nativeRunFinalizations.controllerBootId }) + .from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); + if (owner?.bootId && owner.bootId !== controllerBootId) return false; + // A graceful shutdown relinquishes controller authority just like a + // hot restart. Leaving the old event consumer attached lets its + // finalizer interrupt/suspend Claude while the next server is adopting + // the same turn. + markHandedOffByThisController(run.id); + await detachNativeSessionsForRestart([run.id]); + const recoveryHistoryEntry = JSON.stringify({ + at: now.toISOString(), + restartKind: "graceful", + disposition: "restart_suspended", + reason: signal, + processPid: run.processPid, + processStartedAt: run.processStartedAt?.toISOString() ?? null, + }); + // An accepted result needs finalization, not a runner reattach. Its + // execution lease must still be released: the detached executor will + // not finalize, and the next controller's reconciler skips leased rows. + // A finalizer lease belongs to finalization already in progress here. + const released = await db.transaction(async tx => { + await notifyLiveRunRecovery(tx, { companyId: run.companyId, runId: run.id }); + await notifyReattachHandOver(tx, run.id); + return tx.update(nativeRunFinalizations).set({ + leaseExpiresAt: sql`case when ${nativeRunFinalizations.resultId} is not null + and ${nativeRunFinalizations.leaseOwner} like 'native-finalizer:%' + then ${nativeRunFinalizations.leaseExpiresAt} else ${now.toISOString()}::timestamptz end`, + recoveryState: sql`case when ${nativeRunFinalizations.resultId} is null + then 'awaiting_runner_reattach' else ${nativeRunFinalizations.recoveryState} end`, + recoveryHistory: sql`( + select coalesce(jsonb_agg(item order by ordinal), '[]'::jsonb) + from jsonb_array_elements( + coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb) + || jsonb_build_array(${recoveryHistoryEntry}::jsonb) + ) with ordinality as history(item, ordinal) + where ordinal > greatest( + jsonb_array_length( + coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb) + || jsonb_build_array(${recoveryHistoryEntry}::jsonb) + ) - 20, + 0 + ) + )`, + updatedAt: now, + }) + .where( + and( + eq(nativeRunFinalizations.runId, run.id), + owner?.bootId ? eq(nativeRunFinalizations.controllerBootId, owner.bootId) + : isNull(nativeRunFinalizations.controllerBootId), + ), + ) + .returning({ resultId: nativeRunFinalizations.resultId, leaseOwner: nativeRunFinalizations.leaseOwner }); + }); + const resultAccepted = Boolean(released[0]?.resultId); + const finalizerHeld = resultAccepted && Boolean(released[0]?.leaseOwner?.startsWith("native-finalizer:")); + await appendRunEvent(run, { + eventType: "native.recovery.transition", + stream: "system", + level: "info", + message: finalizerHeld + ? "Server shutdown kept the in-progress finalizer lease of the accepted native result" + : resultAccepted + ? "Server shutdown released the accepted native result for finalization by the next controller" + : "Server shutdown suspended native controller ownership without cancelling provider work", + payload: { + restartKind: "graceful", + signal, + runnerDisposition: finalizerHeld ? "finalizer_in_progress" + : resultAccepted ? "awaiting_result_finalization" : "awaiting_runner_reattach", + processPid: run.processPid, + processGroupId: run.processGroupId, + processStartedAt: run.processStartedAt?.toISOString() ?? null, + retryRunCreated: false, + }, + }); + return true; + } + + async function handOffNativeRunsForShutdown(signal: "SIGINT" | "SIGTERM", now = new Date()) { + const intent = await readHotRestartIntent().catch(() => null); + if (intent && !intent.drainRequired && shouldHonorHotRestartIntentForProcess(intent)) return []; + const nativeController = await currentNativeControllerIdentity(); + const liveRuns = await db.select({ run: heartbeatRuns }).from(heartbeatRuns) + .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)) + .where(and(eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.runtimeMode, "native"), + eq(agents.adapterType, "paperclip_runner"))); + const handedOff: string[] = []; + await Promise.allSettled(liveRuns.map(async ({ run }) => { + if (isNativeRunnerOwnershipHeld(run) || nativeRunsHandedOffEarly.has(run.id)) return; + nativeRunsHandedOffEarly.add(run.id); + try { + if (await suspendNativeRunForShutdown(run, signal, now, nativeController.bootId)) handedOff.push(run.id); + else nativeRunsHandedOffEarly.delete(run.id); + } catch (error) { + nativeRunsHandedOffEarly.delete(run.id); + logger.warn({ err: error, runId: run.id }, "early native run hand-off failed; the shutdown drain retries it"); + } + })); + return handedOff; + } + async function drainRunningRunsForShutdown( signal: "SIGINT" | "SIGTERM", now = new Date(), @@ -1196,89 +1311,14 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD run.runtimeMode === "native" && agent.adapterType === "paperclip_runner" ) { - const [owner] = await db.select({ bootId: nativeRunFinalizations.controllerBootId }) - .from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); - if (owner?.bootId && owner.bootId !== nativeController.bootId) return; - // A graceful shutdown relinquishes controller authority just like a - // hot restart. Leaving the old event consumer attached lets its - // finalizer interrupt/suspend Claude while the next server is adopting - // the same turn. - markRunHandedOffForShutdown(run.id); - await detachNativeSessionsForRestart([run.id]); - const recoveryHistoryEntry = JSON.stringify({ - at: now.toISOString(), - restartKind: "graceful", - disposition: "restart_suspended", - reason: signal, - processPid: run.processPid, - processStartedAt: run.processStartedAt?.toISOString() ?? null, - }); - // An accepted result needs finalization, not a runner reattach. Its - // execution lease must still be released: the detached executor will - // not finalize, and the next controller's reconciler skips leased rows. - // A finalizer lease belongs to finalization already in progress here. - const released = await db.transaction(async tx => { - await notifyLiveRunRecovery(tx, { companyId: run.companyId, runId: run.id }); - return tx.update(nativeRunFinalizations).set({ - leaseExpiresAt: sql`case when ${nativeRunFinalizations.resultId} is not null - and ${nativeRunFinalizations.leaseOwner} like 'native-finalizer:%' - then ${nativeRunFinalizations.leaseExpiresAt} else ${now.toISOString()}::timestamptz end`, - recoveryState: sql`case when ${nativeRunFinalizations.resultId} is null - then 'awaiting_runner_reattach' else ${nativeRunFinalizations.recoveryState} end`, - recoveryHistory: sql`( - select coalesce(jsonb_agg(item order by ordinal), '[]'::jsonb) - from jsonb_array_elements( - coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb) - || jsonb_build_array(${recoveryHistoryEntry}::jsonb) - ) with ordinality as history(item, ordinal) - where ordinal > greatest( - jsonb_array_length( - coalesce(${nativeRunFinalizations.recoveryHistory}, '[]'::jsonb) - || jsonb_build_array(${recoveryHistoryEntry}::jsonb) - ) - 20, - 0 - ) - )`, - updatedAt: now, - }) - .where( - and( - eq(nativeRunFinalizations.runId, run.id), - owner?.bootId ? eq(nativeRunFinalizations.controllerBootId, owner.bootId) - : isNull(nativeRunFinalizations.controllerBootId), - ), - ) - .returning({ resultId: nativeRunFinalizations.resultId, leaseOwner: nativeRunFinalizations.leaseOwner }); - }); - const resultAccepted = Boolean(released[0]?.resultId); - const finalizerHeld = resultAccepted && Boolean(released[0]?.leaseOwner?.startsWith("native-finalizer:")); - await appendRunEvent(run, { - eventType: "native.recovery.transition", - stream: "system", - level: "info", - message: finalizerHeld - ? "Server shutdown kept the in-progress finalizer lease of the accepted native result" - : resultAccepted - ? "Server shutdown released the accepted native result for finalization by the next controller" - : "Server shutdown suspended native controller ownership without cancelling provider work", - payload: { - restartKind: "graceful", - signal, - runnerDisposition: finalizerHeld ? "finalizer_in_progress" - : resultAccepted ? "awaiting_result_finalization" : "awaiting_runner_reattach", - processPid: run.processPid, - processGroupId: run.processGroupId, - processStartedAt: run.processStartedAt?.toISOString() ?? null, - retryRunCreated: false, - }, - }); - restartSuspendedRunIds.push(run.id); + if (nativeRunsHandedOffEarly.has(run.id) + || await suspendNativeRunForShutdown(run, signal, now, nativeController.bootId)) restartSuspendedRunIds.push(run.id); return; } if (agent.adapterType === "paperclip_runner" && run.runtimeMode === "legacy" && run.scopeKind === "issue" && run.issueId && run.executionStage === "preparing" && run.controllerBootId === legacyControllerBootId && await handoffLegacyControllerPreparation(db, run, portableTaskPreparationLeaseId(db, run), - () => markRunHandedOffForShutdown(run.id))) { + () => markHandedOffByThisController(run.id))) { restartSuspendedRunIds.push(run.id); return; } @@ -3100,5 +3140,6 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD recoverActiveExecutions, sweepOrphanedActiveLeases, drainRunningRunsForShutdown, + handOffNativeRunsForShutdown, }; } diff --git a/server/src/services/heartbeat/run-admission.ts b/server/src/services/heartbeat/run-admission.ts index d61a0b48bd0..2bd0e7a95f5 100644 --- a/server/src/services/heartbeat/run-admission.ts +++ b/server/src/services/heartbeat/run-admission.ts @@ -1,3 +1,5 @@ +import { logger } from "../../middleware/logger.js"; + let runAdmissionClosed = false; export function closeProcessRunAdmission(): void { @@ -11,3 +13,234 @@ export function isProcessRunAdmissionClosed(): boolean { export function resetProcessRunAdmissionForTests(): void { runAdmissionClosed = false; } + +export type ReattachOutcome = "reached" | "given_up"; +export const REATTACH_FAILURE_THRESHOLD_MS = 150_000; +export type ReattachPhaseEndReason = "nothing_to_reattach" | "all_settled" | "cap_reached"; +export type ReattachClaimableReason = "handed_over" | "owner_gone" | "lease_expired"; +export type ClaimableReattachRun = { runId: string; reason: ReattachClaimableReason; claimableAt: number }; + +export type ReattachPhaseSummary = { + reason: ReattachPhaseEndReason; + startReason: ReattachClaimableReason | null; + liveRuns: number; + authenticated: number; + reached: number; + reachedAfterThreshold: number; + givenUp: number; + outstanding: number; + maxReachLatencyMs: number | null; + durationMs: number; + capMs: number; + gatedStarts: boolean; + runs: Array<{ runId: string; outcome: ReattachOutcome | "outstanding"; latencyMs: number; authenticatedMs: number | null }>; +}; + +type ReattachPhase = { + startedAt: number; + startReason: ReattachClaimableReason; + capMs: number; + deadline: number; + pending: Set; + claimable: Map; + settled: Array<{ runId: string; outcome: ReattachOutcome; at: number }>; + authenticatedAt: Map; + gatedStarts: boolean; + announced: boolean; + capTimer: ReturnType | null; +}; + +let reattachCapMs = 0; +let reattachEnabledAt = 0; +let reattachPhase: ReattachPhase | null = null; +const joinedReattachRuns = new Set(); +const reattachPhaseEndListeners = new Set<(summary: ReattachPhaseSummary) => void>(); + +export function isReattachPhaseActive(now = Date.now()): boolean { + const phase = reattachPhase; + if (!phase) return false; + if (now >= phase.deadline) { + endReattachPhase("cap_reached", now); + return false; + } + phase.gatedStarts = true; + return true; +} + +export function reattachPhaseRecheckAt(now = Date.now()): Date { + const deadline = reattachPhase?.deadline ?? now; + return new Date(Math.max(now + 1_000, Math.min(deadline, now + 10_000))); +} + +export function enableReattachPhase(capMs: number, now = Date.now()): void { + reattachCapMs = capMs; + reattachEnabledAt = now; +} + +function claimableSince(claimableAt: number, now: number): number { + return Math.min(now, Math.max(reattachEnabledAt, claimableAt)); +} + +export function isReattachPhaseEnabled(): boolean { + return reattachCapMs > 0; +} + +export function hasJoinedReattachPhase(runId: string): boolean { + return joinedReattachRuns.has(runId); +} + +export function joinReattachPhase(runs: readonly ClaimableReattachRun[], options: { announce?: boolean } = {}, + now = Date.now()): string[] { + if (reattachCapMs <= 0) return []; + const fresh = runs.filter(run => !joinedReattachRuns.has(run.runId)); + if (fresh.length === 0) return []; + let phase = reattachPhase; + if (!phase) { + const first = fresh.reduce((earliest, run) => run.claimableAt < earliest.claimableAt ? run : earliest); + phase = { + startedAt: claimableSince(first.claimableAt, now), startReason: first.reason, capMs: reattachCapMs, deadline: 0, + pending: new Set(), claimable: new Map(), settled: [], authenticatedAt: new Map(), gatedStarts: false, + announced: false, capTimer: null, + }; + reattachPhase = phase; + } + for (const run of fresh) { + joinedReattachRuns.add(run.runId); + phase.pending.add(run.runId); + phase.claimable.set(run.runId, { reason: run.reason, at: claimableSince(run.claimableAt, now) }); + } + armReattachCap(phase, Math.max(...fresh.map(run => claimableSince(run.claimableAt, now))) + phase.capMs, now); + if (options.announce !== false) announceReattachPhase(fresh.map(run => run.runId), now); + return fresh.map(run => run.runId); +} + +export function refineReattachClaimable(runs: readonly ClaimableReattachRun[], now = Date.now()): void { + const phase = reattachPhase; + if (!phase) return; + for (const run of runs) { + const current = phase.claimable.get(run.runId); + if (!current) continue; + const at = Math.min(current.at, claimableSince(run.claimableAt, now)); + phase.claimable.set(run.runId, { reason: run.reason, at }); + if (at < phase.startedAt) phase.startedAt = at; + } + const entries = [...phase.claimable.values()]; + const earliest = entries.reduce((first, entry) => entry.at < first.at ? entry : first); + phase.startReason = earliest.reason; + const deadline = Math.max(...entries.map(entry => entry.at)) + phase.capMs; + if (deadline < phase.deadline) { + phase.deadline = 0; + armReattachCap(phase, deadline, now); + } +} + +export function announceReattachPhase(runIds: readonly string[], now = Date.now()): void { + const phase = reattachPhase; + if (!phase) return; + const joined = runIds.filter(runId => phase.claimable.has(runId)); + if (joined.length === 0) return; + const reasons = Object.fromEntries((["handed_over", "owner_gone", "lease_expired"] as const) + .map(reason => [reason, joined.filter(runId => phase.claimable.get(runId)?.reason === reason).length]) + .filter(([, count]) => count)); + const fields = { reason: phase.startReason, reasons, liveRuns: phase.claimable.size, joined: joined.length, + capMs: phase.capMs, capEndsInMs: phase.deadline - now, + claimableAgoMs: Math.max(...joined.map(runId => now - phase.claimable.get(runId)!.at)) }; + if (!phase.announced) { + phase.announced = true; + logger.info(fields, "reattach phase started: live runs of a previous server became claimable; new run starts wait until each is reached or given up"); + } else { + logger.info(fields, "reattach phase grew: more live runs of a previous server became claimable"); + } +} + +function armReattachCap(phase: ReattachPhase, deadline: number, now: number): void { + if (deadline <= phase.deadline) return; + phase.deadline = deadline; + if (phase.capTimer) clearTimeout(phase.capTimer); + phase.capTimer = setTimeout(() => { if (reattachPhase === phase) endReattachPhase("cap_reached"); }, + Math.max(0, deadline - now)); + phase.capTimer.unref?.(); +} + +export function pendingReattachRunIds(): string[] { + return reattachPhase ? [...reattachPhase.pending] : []; +} + +export function markReattachAuthenticated(runId: string, now = Date.now()): void { + const phase = reattachPhase; + if (!phase || !phase.pending.has(runId) || phase.authenticatedAt.has(runId)) return; + phase.authenticatedAt.set(runId, now); + logger.info({ runId, authenticatedMs: now - phase.claimable.get(runId)!.at }, "reattach phase run authenticated"); +} + +export function settleReattachRun(runId: string, outcome: ReattachOutcome, now = Date.now()): void { + const phase = reattachPhase; + if (!phase || !phase.pending.delete(runId)) return; + phase.settled.push({ runId, outcome, at: now }); + const claimable = phase.claimable.get(runId)!; + const authenticatedAt = phase.authenticatedAt.get(runId); + logger.info({ runId, outcome, claimableReason: claimable.reason, latencyMs: now - claimable.at, + authenticatedMs: authenticatedAt === undefined ? null : authenticatedAt - claimable.at, + remaining: phase.pending.size }, "reattach phase run settled"); + if (phase.pending.size === 0) endReattachPhase("all_settled", now); +} + +export function onReattachPhaseEnd(listener: (summary: ReattachPhaseSummary) => void): () => void { + reattachPhaseEndListeners.add(listener); + return () => { reattachPhaseEndListeners.delete(listener); }; +} + +export function reportNothingToReattach(): void { + publishReattachPhaseEnd({ reason: "nothing_to_reattach", startReason: null, liveRuns: 0, authenticated: 0, reached: 0, + reachedAfterThreshold: 0, givenUp: 0, outstanding: 0, maxReachLatencyMs: null, durationMs: 0, capMs: reattachCapMs, + gatedStarts: false, runs: [] }); +} + +function endReattachPhase(reason: ReattachPhaseEndReason, now = Date.now()): void { + const phase = reattachPhase; + if (!phase) return; + reattachPhase = null; + if (phase.capTimer) clearTimeout(phase.capTimer); + const latency = (runId: string, at: number) => at - phase.claimable.get(runId)!.at; + const authenticatedMs = (runId: string) => { + const at = phase.authenticatedAt.get(runId); + return at === undefined ? null : latency(runId, at); + }; + const runs = [ + ...phase.settled.map(run => ({ runId: run.runId, outcome: run.outcome, latencyMs: latency(run.runId, run.at) })), + ...[...phase.pending].map(runId => ({ runId, outcome: "outstanding" as const, latencyMs: latency(runId, now) })), + ].map(run => ({ ...run, authenticatedMs: authenticatedMs(run.runId) })); + const reached = runs.filter(run => run.outcome === "reached"); + publishReattachPhaseEnd({ + reason, + startReason: phase.startReason, + liveRuns: phase.claimable.size, + authenticated: phase.authenticatedAt.size, + reached: reached.length, + reachedAfterThreshold: reached.filter(run => run.latencyMs > REATTACH_FAILURE_THRESHOLD_MS).length, + givenUp: runs.filter(run => run.outcome === "given_up").length, + outstanding: phase.pending.size, + maxReachLatencyMs: reached.length ? Math.max(...reached.map(run => run.latencyMs)) : null, + durationMs: now - phase.startedAt, + capMs: phase.capMs, + gatedStarts: phase.gatedStarts, + runs, + }); +} + +function publishReattachPhaseEnd(summary: ReattachPhaseSummary): void { + if (summary.reason === "cap_reached") logger.warn(summary, "reattach phase ended"); + else logger.info(summary, "reattach phase ended"); + for (const listener of reattachPhaseEndListeners) { + try { listener(summary); } catch (error) { logger.warn({ err: error }, "reattach phase end listener failed"); } + } +} + +export function resetReattachPhaseForTests(): void { + if (reattachPhase?.capTimer) clearTimeout(reattachPhase.capTimer); + reattachPhase = null; + reattachCapMs = 0; + reattachEnabledAt = 0; + joinedReattachRuns.clear(); + reattachPhaseEndListeners.clear(); +} diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index 3ae6522b205..fb4cbeefc52 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -642,6 +642,27 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( } else expect(f.deps.finalizeAgentStatus).not.toHaveBeenCalled(); }); + it.each(["completeRun", "failRun"] as const)("finalizes the run log in %s only after a reattach's deferred log recovery settles", async operation => { + const f = await fixture(); + let finishRecovery!: () => void; + const recovery = new Promise(resolve => { finishRecovery = resolve; }); + let recovered = false; + const output = { ...f.input.output, settleDeferredLogRecovery: () => recovery.then(() => { recovered = true; }) }; + const finalizedAfterRecovery: boolean[] = []; + f.deps.runLogStore.finalize.mockImplementation(async () => { + finalizedAfterRecovery.push(recovered); + return { bytes: 123, sha256: "log-hash", compressed: true }; + }); + const completion = operation === "completeRun" ? f.completion.completeRun({ ...f.input, output }) + : f.completion.failRun({ ...f.failure(new Error("reattach preparation failed")), output }); + await new Promise(resolve => setTimeout(resolve, 50)); + const finalizedEarly = f.deps.runLogStore.finalize.mock.calls.length; + finishRecovery(); + await completion; + expect(finalizedEarly).toBe(0); + expect(finalizedAfterRecovery).toEqual([true]); + }); + it("attributes an adapter stop caused by a lost controller lease", async () => { const f = await fixture(); Object.assign(f.input.adapterResult, { exitCode: null, errorCode: "cancelled", errorMessage: "Grok execution was cancelled" }); diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 233e7925ee1..87f95c21daf 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -113,6 +113,7 @@ interface RunCompletionOutput { handle: RunLogHandle | null; outputProgressState: { pending: { bytes: number } | null }; flushOutputProgress: (options?: { force?: boolean }) => Promise; + settleDeferredLogRecovery?: () => Promise; // Getters in the executor retain output that arrives while finalization awaits I/O. readonly stdoutExcerpt: string; readonly stderrExcerpt: string; @@ -574,6 +575,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC compressed: boolean; } | null = null; if (handle) { + await output.settleDeferredLogRecovery?.(); logSummary = await measureRunPhase("completion.output", () => runLogStore.finalize(handle)); } const finalLogBytes = logSummary?.bytes; @@ -1347,6 +1349,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC } | null = null; if (handle) { try { + await output.settleDeferredLogRecovery?.(); logSummary = await measureRunPhase("completion.output", () => runLogStore.finalize(handle)); } catch (finalizeErr) { logger.warn( diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index fa8ea45b717..edae7168010 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1,3 +1,4 @@ +import { settleReattachRun } from "./run-admission.js"; import { readNativeStartupCleanup } from "../native-runtime/native-startup-cleanup.js"; import { isContextCompactionRun } from "../context-compaction.js"; import { notifyStrandedAgentStatus } from "../live-run-recovery-notifications.js"; @@ -2327,7 +2328,9 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl runId: string, meta: { pid: number; processGroupId: number | null; startedAt: string; targetKind: "local" | "remote" }, ) { - return retryTransientRunWrite(() => persistHeartbeatRunProcessMetadata(db, runId, meta)); + const persisted = await retryTransientRunWrite(() => persistHeartbeatRunProcessMetadata(db, runId, meta)); + settleReattachRun(runId, "reached"); + return persisted; } async function clearDetachedRunWarning(runId: string) { diff --git a/server/src/services/heartbeat/run-output.test.ts b/server/src/services/heartbeat/run-output.test.ts index 2a65a7da867..706b5c55fe1 100644 --- a/server/src/services/heartbeat/run-output.test.ts +++ b/server/src/services/heartbeat/run-output.test.ts @@ -102,7 +102,7 @@ describe("claiming the run log during a storage outage (storage chaos ST1)", () const pending = (update as (t: unknown) => { set: (v: Partial) => unknown })(table); return { set: (values: Partial) => { writes.push(values); return pending.set(values); } }; }; - const appendRunEvent = vi.fn(async () => undefined); + const appendRunEvent = vi.fn(async (_run: unknown, _event: { payload?: { reason?: string } }) => undefined); const authority = vi.fn(async () => undefined); const seen: Array<{ writes: number; events: number; phase: string | undefined; message: string | undefined }> = []; let calls = 0; @@ -397,6 +397,82 @@ describe("object-backed diagnostic recording", () => { expect(row.lastOutputBytes).toBe(10000); expect(writes.filter((write) => write.lastOutputSeq !== undefined)).toHaveLength(2); }); + + it.each(["succeeds", "fails"] as const)("holds the log's finalization until a reattach's deferred re-verification %s", async (outcome) => { + const row = { ...newRunRow, runtimeMode: "native", logStore: "s3_segments", logRef: "company/agent/run", lastOutputSeq: 4 } as Run; + const db = createRunRowDb(row); + let releaseRead!: () => void; + const read = vi.fn(async () => { + await new Promise(resolve => { releaseRead = resolve; }); + if (outcome === "fails") throw new Error("object store read failed"); + return { content: "", nextOffset: 0, eof: true }; + }); + const appendRunEvent = vi.fn(async (_run: unknown, _event: { payload?: { reason?: string } }) => undefined); + const output = createHeartbeatRunOutput(db, { + run: row, runId: row.id, issueId: null, secretKeys: new Set(), + identityRedactor: { chunk: (_stream, chunk) => chunk, redact: value => value }, + runLogStore: { + begin: async () => ({ store: "s3_segments", logRef: "company/agent/run", initialBytes: 4_000_000, initialSequence: 4, + initialSourceCursor: 900 }), + append: async () => 0, enqueue: async () => ({ committed: Promise.resolve(0) }), flush: async () => undefined, + read: read as never, + }, + appendRunEvent: appendRunEvent as never, getCurrentUserRedactionOptions: async () => ({ enabled: false }), + deferNativeLogRecovery: true, + }); + await output.begin(row); + await vi.waitFor(() => expect(read).toHaveBeenCalled()); + let settled = false; + const settling = output.settleDeferredLogRecovery().then(() => { settled = true; }); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(settled).toBe(false); + releaseRead(); + await settling; + expect(settled).toBe(true); + if (outcome === "fails") expect(appendRunEvent.mock.calls.some(([, event]) => event.payload?.reason === "object_log_persistence_failed")).toBe(true); + }); + + it("settles at once when no log re-verification was deferred", async () => { + const row = { ...newRunRow } as Run; + const output = createHeartbeatRunOutput(createRunRowDb(row), { + run: row, runId: row.id, issueId: null, secretKeys: new Set(), + identityRedactor: { chunk: (_stream, chunk) => chunk, redact: value => value }, + runLogStore: { begin: async () => ({ store: "local_file", logRef: "run.ndjson" }), append: async () => 0 } as never, + appendRunEvent: vi.fn(async () => undefined) as never, getCurrentUserRedactionOptions: async () => ({ enabled: false }), + }); + await expect(output.settleDeferredLogRecovery()).resolves.toBeUndefined(); + }); + + it.each([false, true])("lets a reattach start before its existing log is re-verified (deferred %s)", async (deferNativeLogRecovery) => { + const row = { ...newRunRow, runtimeMode: "native", logStore: "s3_segments", logRef: "company/agent/run", lastOutputSeq: 4 } as Run; + const db = createRunRowDb(row); + let releaseRead!: () => void; + const readStarted = vi.fn(); + const read = vi.fn(async () => { + readStarted(); + await new Promise(resolve => { releaseRead = resolve; }); + return { content: "", nextOffset: 0, eof: true }; + }); + const output = createHeartbeatRunOutput(db, { + run: row, runId: row.id, issueId: null, secretKeys: new Set(), + identityRedactor: { chunk: (_stream, chunk) => chunk, redact: value => value }, + runLogStore: { + begin: async () => ({ store: "s3_segments", logRef: "company/agent/run", initialBytes: 4_000_000, initialSequence: 4, + initialSourceCursor: 900 }), + append: async () => 0, enqueue: async () => ({ committed: Promise.resolve(0) }), flush: async () => undefined, + read: read as never, + }, + appendRunEvent: vi.fn(async () => undefined) as never, getCurrentUserRedactionOptions: async () => ({ enabled: false }), + deferNativeLogRecovery, + }); + let begun = false; + const beginning = output.begin(row).then(() => { begun = true; }); + await vi.waitFor(() => expect(readStarted).toHaveBeenCalled()); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(begun).toBe(deferNativeLogRecovery); + releaseRead(); + await beginning.catch(() => undefined); + }); }); describe("secret-bound environment values in local run output", () => { diff --git a/server/src/services/heartbeat/run-output.ts b/server/src/services/heartbeat/run-output.ts index 06163b15cb7..727a0d129d5 100644 --- a/server/src/services/heartbeat/run-output.ts +++ b/server/src/services/heartbeat/run-output.ts @@ -35,6 +35,7 @@ export interface HeartbeatRunOutputInput { getCurrentUserRedactionOptions: () => Promise; assertWriterAuthority?: () => Promise; runtimeLogSecretValues?: readonly string[]; + deferNativeLogRecovery?: boolean; signal?: AbortSignal; } @@ -84,6 +85,7 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) let persistedLogBytes = Number(run.logBytes ?? 0); let progressWrite: Promise | null = null; let recordingFailureReported = false; + let deferredLogRecovery: Promise | null = null; const flushOutputProgress = async (opts?: { force?: boolean }) => { if (progressWrite) { await progressWrite; @@ -292,8 +294,11 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) flush: () => runLogStore.flush!(handle!), }); })(); - if (handle.store === "s3_segments" && run.runtimeMode === "native") - await (await getNativeExporter()).recover(); + if (handle.store === "s3_segments" && run.runtimeMode === "native") { + const recovering = getNativeExporter().then(exporter => exporter.recover()); + if (input.deferNativeLogRecovery) deferredLogRecovery = recovering.catch(reportRecordingFailure); + else await recovering; + } // Output begins before runtime selection persists native mode. Keep the // source-aware callback available without querying native sources for // ordinary legacy output. @@ -350,6 +355,7 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) }, outputProgressState, flushOutputProgress, + settleDeferredLogRecovery: async () => { await deferredLogRecovery; }, // Finalization awaits I/O and can receive more output before persisting excerpts. get stdoutExcerpt() { return stdoutExcerpt; diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 544940c183a..066de14ceda 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -1116,6 +1116,7 @@ export async function claimNativeRestartRecoveries(input: { liveRunExecutions?: Pick, "has">; /** Runs this process handed to the next server (graceful drain or hot restart). */ handedOffForShutdown?: (runId: string) => boolean; + handedOffByThisController?: (runId: string) => boolean; now?: Date; limit?: number; /** Called after each committed claim, before scanning the next candidate. */ @@ -1412,6 +1413,9 @@ export async function claimNativeRestartRecoveries(input: { // A run this process handed to the next server (graceful drain or hot // restart) stays with that server: this process is about to exit. const ownControllerProcess = !nonExecutionOwner && isCurrentNativeControllerProcess(row.coordinator); + if (ownControllerProcess && input.handedOffByThisController?.(row.run.id)) { + return { kind: "blocked", runId: row.run.id, reason: "handed_off_to_next_controller", transitionUnchanged: true } as const; + } if (ownControllerProcess && activeNativeExecutionRunIds.has(row.run.id)) { return { kind: "awaiting_evidence", runId: row.run.id, reason: "current_controller_preparing" } as const; } diff --git a/server/src/services/native-runtime/native-run-log-export.ts b/server/src/services/native-runtime/native-run-log-export.ts index 33d4c678470..6f0aecf1461 100644 --- a/server/src/services/native-runtime/native-run-log-export.ts +++ b/server/src/services/native-runtime/native-run-log-export.ts @@ -368,14 +368,30 @@ export function createNativeRunLogExporter(input: { await input.flush(); } } + let trailingReplayThrough = 0; + let trailingReplay: Promise | null = null; + const serialized = (work: () => Promise) => { + const next = queue.then(work); + queue = next.catch(() => undefined); + return next; + }; + const replayLater = (through: number) => { + trailingReplayThrough = Math.max(trailingReplayThrough, through); + return trailingReplay ??= serialized(async () => { + trailingReplay = null; + await replay(trailingReplayThrough); + }); + }; return { - recover: () => replay(), + recover: () => serialized(() => replay()), async record(sourceCursor: number, chunk: string) { if (!Number.isSafeInteger(sourceCursor) || sourceCursor < 1) throw new Error("invalid_native_run_log_cursor"); const bytes = Buffer.byteLength(chunk); - if (queuedRecords >= 32 || queuedBytes + bytes > MAX_RUN_LOG_RECORD_BYTES) - throw new Error("native_run_log_admission_capacity_exceeded"); + if (queuedRecords >= 32 || queuedBytes + bytes > MAX_RUN_LOG_RECORD_BYTES) { + await replayLater(sourceCursor); + return; + } queuedRecords++; queuedBytes += bytes; const work = queue diff --git a/server/src/services/native-runtime/native-runner-execution-driver.ts b/server/src/services/native-runtime/native-runner-execution-driver.ts index 95eba9c5515..ecf6555e5d0 100644 --- a/server/src/services/native-runtime/native-runner-execution-driver.ts +++ b/server/src/services/native-runtime/native-runner-execution-driver.ts @@ -1,3 +1,4 @@ +import { markReattachAuthenticated } from "../heartbeat/run-admission.js"; import { existsSync } from "node:fs"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { resolvePaperclipRunnerTransport, type PaperclipRunnerTransport } from "@paperclipai/adapter-utils/runner-connectivity"; @@ -319,6 +320,7 @@ export function createNativeRunnerExecutionDriver(input: NativeRunnerExecutionDr "runner.prp.authenticate", () => outbound!.ready, ); + markReattachAuthenticated(input.binding.runId); await input.processIdentityReady; }, { parentName: "runner.session.startup" }, diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 150cccd0938..26553995dbf 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -3709,10 +3709,20 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); it("leaves a run this process handed to the next server for that server", async () => { const fixture = await seedUnlaunchedRemoteClaim("SELF-LAPSED-HANDED-OFF", { liveController: true, leaseExpired: true, event: { eventType: "native.process_start_requested" } }); + const [before] = await fixture.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, fixture.runId)); + const activityBefore = await fixture.db.select({ id: activityLog.id }).from(activityLog).where(eq(activityLog.companyId, companyId)); + const actionsBefore = await fixture.db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) + .where(eq(issueRecoveryActions.companyId, companyId)); expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: await currentNativeControllerIdentity(), restartKind: "hard", runIds: [fixture.runId], liveRunExecutions: new Set(), - handedOffForShutdown: runId => runId === fixture.runId })) - .toEqual([expect.objectContaining({ kind: "awaiting_evidence", reason: "controller_still_alive" })]); + handedOffForShutdown: runId => runId === fixture.runId, handedOffByThisController: runId => runId === fixture.runId })) + .toEqual([expect.objectContaining({ kind: "blocked", reason: "handed_off_to_next_controller" })]); + const [after] = await fixture.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(after).toEqual(before); + expect(await fixture.db.select({ id: activityLog.id }).from(activityLog).where(eq(activityLog.companyId, companyId))) + .toEqual(activityBefore); + expect(await fixture.db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) + .where(eq(issueRecoveryActions.companyId, companyId))).toEqual(actionsBefore); }); it("keeps waiting for a different live controller in this process space", async () => { diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 04339beaeaf..eb678727efe 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -20,6 +20,7 @@ import { trace } from "@opentelemetry/api"; import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; import { configuredEnvironment, configuredEnvironmentProjection } from "../../vendor/paperclip-runner/index.js"; import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest"; +import { enableReattachPhase, isReattachPhaseActive, joinReattachPhase, onReattachPhaseEnd, resetReattachPhaseForTests, type ReattachPhaseSummary } from "../heartbeat/run-admission.js"; import { createSystemRecovery, failSystemRecovery, reserveSystemRecoveryAttempt, SystemRecoveryFailure } from "../system-recovery-state.js"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { once } from "node:events"; @@ -8391,6 +8392,31 @@ describe("native startup cancellation fence", () => { } }); + it.each(["codex_app_server", "opencode_server", "acpx_runtime", "muse_external", "grok_bot_mcp", "openai_dot_mcp"])( + "ends the reattach phase once a reattached %s session is driven by this server again, without the cap or manual settling", async driverKind => { + const current: NativeExecutionInputV1 = { ...execution, + binding: { ...execution.binding, runId: `reattach-reached-${driverKind}` }, + session: { ...execution.session, normalizedSessionId: `reattach-reached-session-${driverKind}`, driverKind } as never }; + const ended: ReattachPhaseSummary[] = []; + try { + onReattachPhaseEnd(summary => ended.push(summary)); + enableReattachPhase(180_000); + joinReattachPhase([{ runId: current.binding.runId, reason: "handed_over", claimableAt: Date.now() }]); + expect(isReattachPhaseActive()).toBe(true); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession({ close: vi.fn(async () => undefined) }); + return { result: { summary: "done" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", + normalizedSessionId: current.session.normalizedSessionId, providerSessionId: "provider", driverKind, driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + await executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: `reattach-reached-${driverKind}` }); + expect(isReattachPhaseActive()).toBe(false); + expect(ended).toEqual([expect.objectContaining({ reason: "all_settled", reached: 1, givenUp: 0, outstanding: 0 })]); + } finally { + resetReattachPhaseForTests(); + } + }); + it.each(["deadline", "stop"])("retires a claimed warm owner when %s interrupts its reuse diagnostic", async mode => { const first: NativeExecutionInputV1 = { ...execution, binding: { ...execution.binding, runId: `reuse-log-first-${mode}`, executionWorkspaceId: `reuse-log-workspace-${mode}` }, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index a43f59d77d5..dc3d0d78106 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,4 +1,5 @@ import { currentControllerProcess } from "../controller-process-identity.js"; +import { settleReattachRun } from "../heartbeat/run-admission.js"; import { remoteRecoveryLeaseMismatch } from "./remote-recovery-lease-mismatch.js"; import { isGovernancePauseStop } from "../governance-pause-stop.js"; import { retryTransientLockContention } from "../transient-lock-contention.js"; @@ -12526,6 +12527,7 @@ async function executePaperclipNativeSessionAttempt( ); }, onSession: async (session) => { + if (session) settleReattachRun(input.execution.binding.runId, "reached"); input.bindAttachmentSession?.(session); if (session) runnerdBackend?.bindDiagnosticSession?.(session); if (session && input.runnerExecutionTarget?.kind === "remote" diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 9128d888876..c5c34a68819 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -33,6 +33,7 @@ import { workspaceRecoverySourceEligible } from "../workspace-recovery-source.js import { PENDING_CONTINUATION_BOUND_MS } from "../recovery-continuation-pacing.js"; import { attributeBlock, BLOCK_ATTRIBUTION_WINDOW_MS, blockedStatusDetailsSql } from "../interruption-block-release.js"; import { ASSIGNED_WORK_CONTINUED_ACTION, taskAutomaticAttempts } from "./task-automatic-attempts.js"; +import { isReattachPhaseActive, reattachPhaseRecheckAt } from "../heartbeat/run-admission.js"; import { ensureFreshContinuation, type FreshContinuationResult } from "../fresh-continuation-fallback.js"; import { canContinueRecoveryRunAutomatically } from "../continuation-cause.js"; import { STRANDED_CONTINUATION_MAX_AGE_MS, STRANDED_KIND, strandedHoldContinuesAutomatically, type StrandedHoldRun } from "./stranded-hold-continuation.js"; @@ -100,7 +101,7 @@ export type AssignedWorkDeps = { export type AssignedWorkCheck = { verdict: AssignedWorkVerdict; - outcome: "ok" | "settling" | "repaired" | "settled_lost_run" | "deferred_rate_limit" | "deferred_cap" | "stale_reported" | "left_to_hold_owner" | "left_to_native_recovery" | "changed" | "enqueue_refused" + outcome: "ok" | "settling" | "repaired" | "settled_lost_run" | "deferred_rate_limit" | "deferred_cap" | "deferred_reattach_phase" | "stale_reported" | "left_to_hold_owner" | "left_to_native_recovery" | "changed" | "enqueue_refused" | "waiting_on_recovery" | "started_saved_work_recovery" | "saved_work_recovery_busy"; retryAt: Date | null; }; @@ -552,13 +553,11 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { if (verdict.kind === "not_applicable" || verdict.kind === "ok") return { verdict, outcome: "ok", retryAt: null }; if (verdict.kind === "settling") return { verdict, outcome: "settling", retryAt: verdict.recheckAt }; if (verdict.stale) return { verdict, outcome: "stale_reported", retryAt: null }; + if (verdict.reason === "saved_work_recovery_overdue") return startSavedWorkRecovery(issue, facts, verdict, at); + if (isReattachPhaseActive()) return { verdict, outcome: "deferred_reattach_phase", retryAt: reattachPhaseRecheckAt(at.getTime()) }; if (verdict.reason === "blocked_with_live_run") { return { verdict, outcome: await liftBlockBehindLiveRun(issue, at) ? "repaired" : "changed", retryAt: null }; } - // The saved files come first: start their overdue recovery. Its hold - // refuses any wake until it completes, and its completion continues the - // task (or, if the files cannot be recovered, offers a working Retry). - if (verdict.reason === "saved_work_recovery_overdue") return startSavedWorkRecovery(issue, facts, verdict, at); // A task blocked under an active recovery hold is re-continued by the // hold's owner (the fresh-continuation fallback, same per-task cap). The // sweep reports it and repairs only blocks that have no hold at all, so diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 820eddc7225..16a8f0e517d 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -1130,7 +1130,11 @@ task finishes with no Retry and no run ends with Before starting, the harness compares the image's `io.paperclip.runner.content-id` label with the content ID of this checkout (`pnpm test:e2e:runner:image-id`, -which hashes every runner input) and refuses to run on a mismatch. To get an +which hashes every runner input) and refuses to run on a mismatch. For a local +test of a branch that changes only controller-side TypeScript, set +`PAPERCLIP_CHAOS_TEST_ACCEPT_IMAGE_CONTENT_ID` to the image's exact content ID +to run against that existing image; the harness prints a `TEST OVERRIDE` warning +and the timeline keeps both content IDs. Such a wave is not image qualification. To get an image for the current master, dispatch a narrow image build and reject its paid gate: diff --git a/tests/restart-chaos/daytona-image.test.ts b/tests/restart-chaos/daytona-image.test.ts index 3c437ff875a..87755ac4917 100644 --- a/tests/restart-chaos/daytona-image.test.ts +++ b/tests/restart-chaos/daytona-image.test.ts @@ -2,7 +2,7 @@ import { createHash } from "node:crypto"; import { chmod, mkdir, mkdtemp, readFile, readlink, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { RUNNER_REVISION_LABEL, assertControllerMatchesImage, assertProviderPackMismatch, copyImageToDir, extractFromLayers, prepareDaytonaImageArtifacts, normalizeLayerPath, walkTar, type LayerSource } from "./daytona-image.js"; type Entry = @@ -191,6 +191,21 @@ describe("assertControllerMatchesImage", () => { it("names both content IDs and the image revision when they differ", async () => { await expect(assertControllerMatchesImage(image, "/unused", "e".repeat(64))).rejects.toThrow(/does not match the Daytona image[\s\S]*d{40}[\s\S]*e{64}/); }); + it("accepts a mismatch only for the exact image content ID named by the test override", async () => { + const previous = process.env.PAPERCLIP_CHAOS_TEST_ACCEPT_IMAGE_CONTENT_ID; + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + try { + process.env.PAPERCLIP_CHAOS_TEST_ACCEPT_IMAGE_CONTENT_ID = "f".repeat(64); + await expect(assertControllerMatchesImage(image, "/unused", "e".repeat(64))).rejects.toThrow(/does not match the Daytona image/); + process.env.PAPERCLIP_CHAOS_TEST_ACCEPT_IMAGE_CONTENT_ID = "c".repeat(64); + await expect(assertControllerMatchesImage(image, "/unused", "e".repeat(64))).resolves.toEqual({ localContentId: "e".repeat(64) }); + expect(warn).toHaveBeenCalledWith(expect.stringMatching(/TEST OVERRIDE[\s\S]*c{64}[\s\S]*e{64}/)); + } finally { + warn.mockRestore(); + if (previous === undefined) delete process.env.PAPERCLIP_CHAOS_TEST_ACCEPT_IMAGE_CONTENT_ID; + else process.env.PAPERCLIP_CHAOS_TEST_ACCEPT_IMAGE_CONTENT_ID = previous; + } + }); }); describe("assertProviderPackMismatch", () => { diff --git a/tests/restart-chaos/daytona-image.ts b/tests/restart-chaos/daytona-image.ts index 4193402f36e..7a987acaaf2 100644 --- a/tests/restart-chaos/daytona-image.ts +++ b/tests/restart-chaos/daytona-image.ts @@ -465,7 +465,14 @@ export async function prepareDaytonaImageArtifacts(input: { image: string; cache export async function assertControllerMatchesImage(artifacts: Pick, repositoryRoot: string, localContentId?: string): Promise<{ localContentId: string }> { const computed = localContentId ?? await (await import("../runner-e2e/daytona-image-content.js")).computeDaytonaImageContentId({ repositoryRoot }); - if (computed !== artifacts.contentId) { + if (computed !== artifacts.contentId && process.env.PAPERCLIP_CHAOS_TEST_ACCEPT_IMAGE_CONTENT_ID === artifacts.contentId) { + console.warn([ + "TEST OVERRIDE: running a controller whose runner source differs from the Daytona image.", + ` image ${artifacts.image}`, + ` image content ID ${artifacts.contentId} (source revision ${artifacts.sourceRevision})`, + ` this checkout's content ID ${computed}`, + ].join("\n")); + } else if (computed !== artifacts.contentId) { throw new Error([ "Controller runner source does not match the Daytona image.", ` image ${artifacts.image}`,