diff --git a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts index 96ce858744c..67f87267f19 100644 --- a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts @@ -29,6 +29,7 @@ import { currentProcessSpaceHash } from "../controller-process-identity.js"; import { registerControllerInstanceDb, resetControllerInstanceLeaseForTests } from "../controller-instance-lease.js"; import { readProcessStartedAt } from "../hot-restart.js"; import { PROCESS_START_REQUESTED } from "../native-process-start-events.js"; +import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable discovery and import", () => { let database: Awaited>; @@ -704,23 +705,16 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d } finally { resetControllerInstanceLeaseForTests(); } }); - async function parkFollowUpBehindOutgoingServer(waitingFor?: (owner: { runId: string; controllerBootId: string }) => { runId: string; controllerBootId: string }) { + async function followUpBehindOutgoingServer() { const seeded = await seedWarmFollowUpOnOldServer(); - const { fixture, current, identity, recordOwner, runId } = seeded; + const { fixture, current, identity, recordOwner } = seeded; const outgoingBoot = fixture.owner.bootId; await recordOwner({ controllerBootId: outgoingBoot, controllerPid: 7, controllerHostHash: "outgoing-container", controllerProcessStartedAt: new Date(Date.now() - 600_000), leaseOwner: null, leaseExpiresAt: null }); await db.insert(controllerInstances).values({ bootId: outgoingBoot, processSpace: "outgoing-container", pid: 7, leaseExpiresAt: new Date(Date.now() + 90_000) }); - await db.update(nativeRunFinalizations).set({ controllerBootId: randomUUID() }).where(eq(nativeRunFinalizations.runId, runId)); const wait = (checkpointedAt: Date | null) => waitForLiveWarmSessionOwner({ db, execution: current, priorRunId: fixture.scope.runId, checkpointedAt: async () => checkpointedAt }); - const waiting = await wait(null).catch(error => error); - expect(waiting).toBeInstanceOf(NativeControllerRecoveryWaitingError); - expect(waiting.owner).toEqual({ runId: fixture.scope.runId, controllerBootId: outgoingBoot }); - const parkedAt = Date.now(); - await deferNativeControllerRecovery(db, { ...current.binding, retryAt: waiting.retryAt, - waitingForOwner: waitingFor ? waitingFor(waiting.owner) : waiting.owner }); const outgoingRoot = join(root, `${fixture.scope.runId}-outgoing`); (await loadNativeRunnerCheckpoint(db, { companyId, issueId: fixture.scope.issueId, runId: fixture.scope.runId }))! .restoreSessionDirectory(outgoingRoot); @@ -738,10 +732,215 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d await saveSuspend(); return new Date(); }; - return { ...seeded, wait, waiting: waiting as NativeControllerRecoveryWaitingError, parkedAt, outgoing, - fenceHarnessBackup, saveSuspend, closeIdleSession }; + const savedRecords = async () => (await loadNativeRunnerCheckpoint(db, + { companyId, issueId: fixture.scope.issueId, runId: fixture.scope.runId }))?.records; + const savedCommands = async () => { + const restoredRoot = join(root, `${seeded.runId}-${randomUUID()}`); + (await loadNativeRunnerCheckpoint(db, { companyId, issueId: fixture.scope.issueId, runId: fixture.scope.runId }))! + .restoreSessionDirectory(restoredRoot); + return JSON.parse(await readFile(join(restoredRoot, "control-plane", "control-plane-state.json"), "utf8")).commands; + }; + return { ...seeded, outgoingBoot, wait, outgoing, fenceHarnessBackup, saveSuspend, closeIdleSession, savedRecords, savedCommands }; } + async function parkFollowUpBehindOutgoingServer(waitingFor?: (owner: { runId: string; controllerBootId: string }) => { runId: string; controllerBootId: string } | undefined) { + const behind = await followUpBehindOutgoingServer(); + const { fixture, current, runId, wait, outgoingBoot } = behind; + await db.update(nativeRunFinalizations).set({ controllerBootId: randomUUID() }).where(eq(nativeRunFinalizations.runId, runId)); + const waiting = await wait(null).catch(error => error); + expect(waiting).toBeInstanceOf(NativeControllerRecoveryWaitingError); + expect(waiting.owner).toEqual({ runId: fixture.scope.runId, controllerBootId: outgoingBoot }); + const parkedAt = Date.now(); + await deferNativeControllerRecovery(db, { ...current.binding, retryAt: waiting.retryAt, + waitingForOwner: waitingFor ? waitingFor(waiting.owner) : waiting.owner }); + return { ...behind, waiting: waiting as NativeControllerRecoveryWaitingError, parkedAt }; + } + + async function prepareFollowUpOnIncomingServer(runId: string, issueId: string, run: Partial = {}) { + await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + await db.update(heartbeatRuns).set({ runtimeMode: "legacy", issueId, nativeIssueId: null, nativeSessionId: null, + executionStage: "preparing", controllerBootId: randomUUID(), controllerLeaseExpiresAt: new Date(Date.now() + 60_000), ...run }) + .where(eq(heartbeatRuns.id, runId)); + } + + it("the outgoing server checkpoints and closes its idle warm session while the incoming server still prepares the follow-up, before it selects native execution", async () => { + registerControllerInstanceDb(db); + const behind = await followUpBehindOutgoingServer(); + try { + const { fixture, runId, closeIdleSession, savedCommands } = behind; + await prepareFollowUpOnIncomingServer(runId, fixture.scope.issueId); + await closeIdleSession(); + expect(await savedCommands()).toContainEqual(expect.objectContaining({ type: "runner.suspend" })); + } finally { + await behind.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it("holds the incoming server's native selection of the preparing follow-up until the outgoing server's save commits", async () => { + registerControllerInstanceDb(db); + const behind = await followUpBehindOutgoingServer(); + try { + const { fixture, runId, saveSuspend } = behind; + await prepareFollowUpOnIncomingServer(runId, fixture.scope.issueId); + let selection!: Promise; + let selectionSettled = false; + await db.transaction(async tx => { + await assertNativeRunnerCheckpointOwner(tx, fixture.owner, false); + selection = db.transaction(async selectTx => { + await selectTx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update"); + await selectTx.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: fixture.scope.issueId, + nativeSessionId: fixture.scope.normalizedSessionId, executionStage: null, controllerBootId: null, controllerLeaseExpiresAt: null }) + .where(eq(heartbeatRuns.id, runId)); + await selectTx.insert(nativeRunFinalizations).values({ companyId, issueId: fixture.scope.issueId, runId, phase: "observed" }); + }); + void selection.finally(() => { selectionSettled = true; }).catch(() => undefined); + await vi.waitFor(async () => { + const [blocked] = await db.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' and query ilike '%heartbeat_runs%'`); + expect(blocked.count).toBeGreaterThan(0); + }, { timeout: 1_500, interval: 20 }); + expect(selectionSettled).toBe(false); + }); + await selection; + await expect(saveSuspend()).resolves.toBeUndefined(); + } finally { + await behind.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it("accepts the outgoing server's save that waited while the incoming server selected native execution for the preparing follow-up", async () => { + registerControllerInstanceDb(db); + const behind = await followUpBehindOutgoingServer(); + try { + const { fixture, runId, saveSuspend, savedCommands } = behind; + await prepareFollowUpOnIncomingServer(runId, fixture.scope.issueId); + let fence!: Promise; + await db.transaction(async selectTx => { + await selectTx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update"); + fence = db.transaction(async fenceTx => assertNativeRunnerCheckpointOwner(fenceTx, fixture.owner, false)); + void fence.catch(() => undefined); + await vi.waitFor(async () => { + const [blocked] = await db.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' and query ilike '%heartbeat_runs%'`); + expect(blocked.count).toBeGreaterThan(0); + }, { timeout: 1_500, interval: 20 }); + await selectTx.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: fixture.scope.issueId, + nativeSessionId: fixture.scope.normalizedSessionId, executionStage: null, controllerBootId: null, controllerLeaseExpiresAt: null }) + .where(eq(heartbeatRuns.id, runId)); + await selectTx.insert(nativeRunFinalizations).values({ companyId, issueId: fixture.scope.issueId, runId, phase: "observed" }); + }); + await expect(fence).resolves.toBeUndefined(); + await saveSuspend(); + expect(await savedCommands()).toContainEqual(expect.objectContaining({ type: "runner.suspend" })); + } finally { + await behind.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it.each(["expired-controller", "dispatching", "own-process", "own-process-event", "other-issue"] as const)( + "refuses the outgoing server's idle checkpoint when the incoming server's preparing follow-up is no longer unstarted: %s", async scenario => { + registerControllerInstanceDb(db); + const behind = await followUpBehindOutgoingServer(); + try { + const { fixture, runId, fenceHarnessBackup, saveSuspend, savedRecords } = behind; + await prepareFollowUpOnIncomingServer(runId, fixture.scope.issueId, scenario === "expired-controller" + ? { controllerLeaseExpiresAt: new Date(Date.now() - 1_000) } : scenario === "dispatching" ? { executionStage: "dispatching" } + : scenario === "own-process" ? { processPid: 4242, processGroupId: 4242, processStartedAt: new Date() } + : scenario === "other-issue" ? { issueId: null } : {}); + if (scenario === "own-process-event") { + await appendHeartbeatRunEvent(db, { companyId, runId, agentId, eventType: PROCESS_START_REQUESTED }); + } + const before = await savedRecords(); + await expect(fenceHarnessBackup()).rejects.toThrow("native_runner_checkpoint_owner_lost"); + await expect(saveSuspend()).rejects.toThrow("native_runner_checkpoint_owner_lost"); + expect(await savedRecords()).toEqual(before); + } finally { + await behind.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it.each(["unclaimed", "inherited-process"] as const)( + "the outgoing server checkpoints and closes its idle warm session while the incoming server's follow-up is selected but has not checked the owner yet, and the follow-up continues at that check: %s", async scenario => { + registerControllerInstanceDb(db); + const behind = await followUpBehindOutgoingServer(); + try { + const { fixture, runId, wait, closeIdleSession, savedCommands } = behind; + if (scenario === "inherited-process") { + const [successor] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + await db.update(heartbeatRuns).set({ processPid: 4242, processGroupId: 4242, processStartedAt: new Date(), + runnerProfileJson: { ...successor!.runnerProfileJson, inheritedProcessIdentityRunId: fixture.scope.runId } }) + .where(eq(heartbeatRuns.id, runId)); + } + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0]) + .toMatchObject({ phase: "observed", attempt: 0, leaseOwner: null, resultId: null, failureCode: null, recoveryHistory: [] }); + const checkpointedAt = await closeIdleSession(); + expect(await savedCommands()).toContainEqual(expect.objectContaining({ type: "runner.suspend" })); + await behind.outgoing.stop(); + await expect(wait(checkpointedAt)).resolves.toBeUndefined(); + } finally { + await behind.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it.each(["own-process", "other-run-process", "own-process-event", "own-checkpoint", "other-session", "admitted"] as const)( + "refuses the outgoing server's idle checkpoint when the selected follow-up is no longer unstarted: %s", async scenario => { + registerControllerInstanceDb(db); + const behind = await followUpBehindOutgoingServer(); + try { + const { fixture, runId, fenceHarnessBackup, saveSuspend, savedRecords } = behind; + const [successor] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + if (scenario === "own-process") { + await db.update(heartbeatRuns).set({ processPid: 4242, processGroupId: 4242, processStartedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + } else if (scenario === "other-run-process") { + await db.update(heartbeatRuns).set({ processPid: 4242, processGroupId: 4242, processStartedAt: new Date(), + runnerProfileJson: { ...successor!.runnerProfileJson, inheritedProcessIdentityRunId: randomUUID() } }).where(eq(heartbeatRuns.id, runId)); + } else if (scenario === "own-process-event") { + await appendHeartbeatRunEvent(db, { companyId, runId, agentId, eventType: PROCESS_START_REQUESTED }); + } else if (scenario === "own-checkpoint") { + await db.insert(nativeRunnerCheckpoints).values({ companyId, issueId: fixture.scope.issueId, runId, part: "header", material: {} }); + } else if (scenario === "other-session") { + await db.update(heartbeatRuns).set({ nativeSessionId: randomUUID() }).where(eq(heartbeatRuns.id, runId)); + } else { + await db.update(nativeRunFinalizations).set({ attempt: 1, leaseOwner: randomUUID(), controllerBootId: randomUUID(), + leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, runId)); + } + const before = await savedRecords(); + await expect(fenceHarnessBackup()).rejects.toThrow("native_runner_checkpoint_owner_lost"); + await expect(saveSuspend()).rejects.toThrow("native_runner_checkpoint_owner_lost"); + expect(await savedRecords()).toEqual(before); + } finally { + await behind.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it.each(["parked", "re-claimed", "re-claimed-by-previous-release"] as const)( + "the outgoing server checkpoints and closes its idle warm session while the follow-up waits without naming its owner: %s", async scenario => { + registerControllerInstanceDb(db); + const parked = await parkFollowUpBehindOutgoingServer(() => undefined); + try { + const { runId, waiting, closeIdleSession, savedCommands } = parked; + if (scenario !== "parked") { + expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "incoming", runIds: [runId], now: waiting.retryAt })).toHaveLength(1); + } + if (scenario === "re-claimed-by-previous-release") { + await db.execute(sql`update native_run_finalizations set recovery_history = jsonb_set(recovery_history, + array[(jsonb_array_length(recovery_history) - 1)::text, 'evidence'], (recovery_history->-1->'evidence') - 'waitingForOwner') + where run_id = ${runId}`); + } + await closeIdleSession(); + expect(await savedCommands()).toContainEqual(expect.objectContaining({ type: "runner.suspend" })); + } finally { + await parked.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + it("the outgoing server checkpoints and closes its idle warm session while the incoming server's follow-up waits for it, and the follow-up resumes within one recheck", async () => { registerControllerInstanceDb(db); const parked = await parkFollowUpBehindOutgoingServer(); @@ -793,6 +992,63 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d expect(claimSettled).toBe(false); }); expect(await claim).toHaveLength(1); + await expect(parked.saveSuspend()).resolves.toBeUndefined(); + } finally { + await parked.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it("the outgoing server checkpoints and closes its idle warm session while the follow-up is re-claimed for its next recheck, and the follow-up resumes within that recheck", async () => { + registerControllerInstanceDb(db); + const parked = await parkFollowUpBehindOutgoingServer(); + try { + const { fixture, runId, wait, waiting, parkedAt, outgoing, closeIdleSession } = parked; + expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "incoming", runIds: [runId], now: waiting.retryAt })).toHaveLength(1); + const [claimed] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(claimed).toMatchObject({ phase: "observed", attempt: 0, resultId: null, failureDetail: null }); + expect(claimed!.leaseOwner).toBeTruthy(); + const [successor] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(successor).toMatchObject({ status: "running", processPid: null, processGroupId: null, processStartedAt: null }); + + const checkpointedAt = await closeIdleSession(); + const saved = await loadNativeRunnerCheckpoint(db, { companyId, issueId: fixture.scope.issueId, runId: fixture.scope.runId }); + const restoredRoot = join(root, `${runId}-claimed-idle-close`); + saved!.restoreSessionDirectory(restoredRoot); + const restored = JSON.parse(await readFile(join(restoredRoot, "control-plane", "control-plane-state.json"), "utf8")); + expect(restored.commands).toContainEqual(expect.objectContaining({ type: "runner.suspend" })); + await outgoing.stop(); + + await expect(wait(checkpointedAt)).resolves.toBeUndefined(); + const waitedMs = Math.max(waiting.retryAt.getTime(), checkpointedAt.getTime()) - parkedAt; + expect(waitedMs).toBeLessThanOrEqual(5_000); + } finally { + await parked.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it("holds the re-claimed follow-up's process start until the outgoing server's save commits", async () => { + registerControllerInstanceDb(db); + const parked = await parkFollowUpBehindOutgoingServer(); + try { + const { fixture, runId, waiting } = parked; + expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "incoming", runIds: [runId], now: waiting.retryAt })).toHaveLength(1); + let start!: Promise; + let startSettled = false; + await db.transaction(async tx => { + await assertNativeRunnerCheckpointOwner(tx, fixture.owner, false); + start = appendHeartbeatRunEvent(db, { companyId, runId, agentId, eventType: PROCESS_START_REQUESTED, + stream: "system", level: "info", message: "Native execution requested." }); + void start.finally(() => { startSettled = true; }).catch(() => undefined); + await vi.waitFor(async () => { + const [blocked] = await db.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' and query ilike '%heartbeat_runs%'`); + expect(blocked.count).toBeGreaterThan(0); + }, { timeout: 1_500, interval: 20 }); + expect(startSettled).toBe(false); + }); + await start; await expect(parked.saveSuspend()).rejects.toThrow("native_runner_checkpoint_owner_lost"); } finally { await parked.outgoing.stop(); @@ -800,21 +1056,108 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d } }); - it.each(["own-process", "own-process-event", "own-checkpoint", "claimed", "other-owner-boot", "other-owner-run"] as const)( - "refuses the outgoing server's idle checkpoint when the parked follow-up is no longer waiting on it: %s", async scenario => { + it("lets a claim that holds the follow-up's finalization lock its run row while the outgoing server's save waits", async () => { + registerControllerInstanceDb(db); + const parked = await parkFollowUpBehindOutgoingServer(); + try { + const { fixture, runId, waiting } = parked; + expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "incoming", runIds: [runId], now: waiting.retryAt })).toHaveLength(1); + let fence!: Promise; + await db.transaction(async tx => { + await tx.execute(sql`select set_config('lock_timeout', '2000', true)`); + await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)).for("update"); + fence = db.transaction(async fenceTx => assertNativeRunnerCheckpointOwner(fenceTx, fixture.owner, false)); + void fence.catch(() => undefined); + await vi.waitFor(async () => { + const [blocked] = await db.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' and query ilike '%native_run_finalizations%'`); + expect(blocked.count).toBeGreaterThan(0); + }, { timeout: 1_500, interval: 20 }); + await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update"); + }); + await expect(fence).resolves.toBeUndefined(); + } finally { + await parked.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it("refuses the outgoing server's save that waited on the re-claimed follow-up's uncommitted process start", async () => { + registerControllerInstanceDb(db); + const parked = await parkFollowUpBehindOutgoingServer(); + try { + const { fixture, runId, waiting } = parked; + expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "incoming", runIds: [runId], now: waiting.retryAt })).toHaveLength(1); + let release!: () => void; + const released = new Promise(resolve => { release = resolve; }); + let started!: () => void; + const startWritten = new Promise(resolve => { started = resolve; }); + const start = db.transaction(async tx => { + await appendHeartbeatRunEvent(tx as unknown as typeof db, { companyId, runId, agentId, eventType: PROCESS_START_REQUESTED, + stream: "system", level: "info", message: "Native execution requested." }); + started(); + await released; + }); + await startWritten; + const fence = db.transaction(async tx => assertNativeRunnerCheckpointOwner(tx, fixture.owner, false)); + void fence.catch(() => undefined); + await vi.waitFor(async () => { + const [blocked] = await db.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' and query ilike '%heartbeat_runs%'`); + expect(blocked.count).toBeGreaterThan(0); + }, { timeout: 1_500, interval: 20 }); + release(); + await start; + await expect(fence).rejects.toThrow("native_runner_checkpoint_owner_lost"); + } finally { + await parked.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it.each(["own-process", "own-process-event", "own-checkpoint", "other-owner-boot", "other-owner-run", "claim-expired", "attempt-started"] as const)( + "refuses the outgoing server's idle checkpoint when the re-claimed follow-up is no longer waiting on it: %s", async scenario => { registerControllerInstanceDb(db); const parked = await parkFollowUpBehindOutgoingServer(owner => scenario === "other-owner-boot" ? { ...owner, controllerBootId: randomUUID() } : scenario === "other-owner-run" ? { ...owner, runId: randomUUID() } : owner); try { const { fixture, runId, waiting, fenceHarnessBackup, saveSuspend } = parked; + expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "incoming", runIds: [runId], now: waiting.retryAt })).toHaveLength(1); + if (scenario === "own-process") { + await db.update(heartbeatRuns).set({ processPid: 4242, processGroupId: 4242, processStartedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + } else if (scenario === "own-process-event") { + await appendHeartbeatRunEvent(db, { companyId, runId, agentId, eventType: PROCESS_START_REQUESTED }); + } else if (scenario === "own-checkpoint") { + await db.insert(nativeRunnerCheckpoints).values({ companyId, issueId: fixture.scope.issueId, runId, part: "header", material: {} }); + } else if (scenario === "claim-expired") { + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, runId)); + } else if (scenario === "attempt-started") { + await db.update(nativeRunFinalizations).set({ attempt: 1 }).where(eq(nativeRunFinalizations.runId, runId)); + } + const before = await loadNativeRunnerCheckpoint(db, { companyId, issueId: fixture.scope.issueId, runId: fixture.scope.runId }); + await expect(fenceHarnessBackup()).rejects.toThrow("native_runner_checkpoint_owner_lost"); + await expect(saveSuspend()).rejects.toThrow("native_runner_checkpoint_owner_lost"); + expect((await loadNativeRunnerCheckpoint(db, { companyId, issueId: fixture.scope.issueId, runId: fixture.scope.runId }))?.records) + .toEqual(before?.records); + } finally { + await parked.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it.each(["own-process", "own-process-event", "own-checkpoint", "other-owner-boot", "other-owner-run"] as const)( + "refuses the outgoing server's idle checkpoint when the parked follow-up is no longer waiting on it: %s", async scenario => { + registerControllerInstanceDb(db); + const parked = await parkFollowUpBehindOutgoingServer(owner => scenario === "other-owner-boot" + ? { ...owner, controllerBootId: randomUUID() } : scenario === "other-owner-run" ? { ...owner, runId: randomUUID() } : owner); + try { + const { fixture, runId, fenceHarnessBackup, saveSuspend } = parked; if (scenario === "own-process") { await db.update(heartbeatRuns).set({ processPid: 4242, processGroupId: 4242, processStartedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); } else if (scenario === "own-process-event") { await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 1, eventType: PROCESS_START_REQUESTED }); } else if (scenario === "own-checkpoint") { await db.insert(nativeRunnerCheckpoints).values({ companyId, issueId: fixture.scope.issueId, runId, part: "header", material: {} }); - } else if (scenario === "claimed") { - expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "incoming", runIds: [runId], now: waiting.retryAt })).toHaveLength(1); } const before = await loadNativeRunnerCheckpoint(db, { companyId, issueId: fixture.scope.issueId, runId: fixture.scope.runId }); await expect(fenceHarnessBackup()).rejects.toThrow("native_runner_checkpoint_owner_lost"); diff --git a/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts b/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts index d3d6a731189..caf22cf696d 100644 --- a/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts +++ b/server/src/services/native-runtime/native-harness-backup-storage.integration.test.ts @@ -819,7 +819,23 @@ describeEmbeddedPostgres("durable native provider backups", () => { await expect(persistNativeHarnessBackup(f)).rejects.toThrow("native_harness_storage_owner_superseded"); expect(await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest))).toEqual([]); }); - it.each(["foreign_preparation", "expired_preparation", "dispatching_successor", "foreign_native"])("keeps the selected backup when a released owner has a %s", async state => { + it("replaces the selected backup when a released owner's follow-up is still preparing on another server", async () => { + const f = await fixture(); await persistNativeHarnessBackup(f); await commitWarmRun(f); + const [before] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); + const nextRunId = randomUUID(); + await f.db.insert(heartbeatRuns).values({ id: nextRunId, companyId: f.companyId, agentId: environment.current.agentId, + issueId: f.issueId, status: "running", runtimeMode: "legacy", executionStage: "preparing", + controllerBootId: randomUUID(), controllerLeaseExpiresAt: new Date(Date.now() + 120_000) }); + await f.db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, f.issueId)); + const owner = await nativeHarnessBackupOwner(f.db, { companyId: f.companyId, issueId: f.issueId, + runId: f.runId, bootId: environment.current.successor.bootId }); + await persistNativeHarnessBackup({ ...f, owner }); + const current = (await f.db.select().from(nativeHarnessBackups).where(and(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest), + eq(nativeHarnessBackups.status, "current")))).map(row => row.id); + expect(current).toHaveLength(1); + expect(current).not.toEqual([before.id]); + }); + it.each(["expired_preparation", "dispatching_successor", "foreign_native"])("keeps the selected backup when a released owner has a %s", async state => { const f = await fixture(); await persistNativeHarnessBackup(f); await commitWarmRun(f); const [before] = await f.db.select().from(nativeHarnessBackups).where(eq(nativeHarnessBackups.scopeDigest, f.scopeDigest)); const nextRunId = randomUUID(); diff --git a/server/src/services/native-runtime/native-runner-checkpoint.ts b/server/src/services/native-runtime/native-runner-checkpoint.ts index 4a93568dc75..136c3bca4e0 100644 --- a/server/src/services/native-runtime/native-runner-checkpoint.ts +++ b/server/src/services/native-runtime/native-runner-checkpoint.ts @@ -2,7 +2,7 @@ import { readNativeCheckpointRetirement } from "./native-runner-checkpoint-retir import { NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js"; import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; import { createHash } from "node:crypto"; -import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql, type SQL } from "drizzle-orm"; +import { and, eq, exists, gt, isNotNull, isNull, lte, inArray, notExists, or, sql, type SQL } from "drizzle-orm"; import { checkpointDb, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, primeDatabaseWorkTransactionId, type Db } from "@paperclipai/db"; import { inspectDurablePrpCheckpointHeader, inspectUnauthenticatedBootstrapCheckpoint, inspectUnclaimedBootstrapCheckpoint, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, type DurablePrpControlPlane, type DurableRecoveryIdentity } from "../../vendor/paperclip-runner/index.js"; import { createLocalEncryptedCipher, deriveLocalEncryptedInstanceSecret } from "../../secrets/local-encrypted-provider.js"; @@ -11,6 +11,7 @@ import { assertNativeSourceCheckpointConsumer, lockSourceRecoveryCoordinators } import { runnerdStateHasNoProviderTurn, runnerdStateIsIncompleteBootstrap } from "./native-bootstrap-proof.js"; import { NATIVE_RETAINED_OWNER_WAIT } from "./native-recovery-wait-codes.js"; import { OWN_PROCESS_START_EVENT_TYPES } from "../native-process-start-events.js"; +import { SAME_RUN_RESUME_DISPOSITION } from "./same-run-resume-disposition.js"; type Binding = { companyId: string; issueId: string; runId: string }; export const NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES = 1024 * 1024; @@ -118,35 +119,74 @@ export async function validateNativeRunnerCheckpointOwner( eq(heartbeatRuns.issueId, owner.issueId), inArray(heartbeatRuns.status, ["queued", "scheduled_retry"]), notExists(tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, heartbeatRuns.id))))) : []; - const [parkedSuccessor] = !successor && !preparingSuccessor && !waitingSuccessor && task?.executionRunId && task.executionRunId !== owner.runId - ? await parkedSuccessorWaitingForOwner(tx, owner, task.executionRunId) : []; + const [unstartedSuccessor] = !successor && !preparingSuccessor && !waitingSuccessor && task?.executionRunId && task.executionRunId !== owner.runId + ? await unstartedSuccessorOfOwner(tx, owner, task.executionRunId) : []; if (!current.resultId || newWork || !task - || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor && !waitingSuccessor && !parkedSuccessor)) { + || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor && !waitingSuccessor && !unstartedSuccessor)) { throw new Error("native_runner_checkpoint_owner_lost"); } } } -function parkedSuccessorWaitingForOwner(tx: CheckpointTransaction, owner: NativeRunnerCheckpointFenceOwner, successorRunId: string) { - return tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations) - .innerJoin(heartbeatRuns, eq(heartbeatRuns.id, nativeRunFinalizations.runId)).where(and( - eq(nativeRunFinalizations.runId, successorRunId), eq(nativeRunFinalizations.companyId, owner.companyId), - eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.phase, "retryable_failure"), - eq(nativeRunFinalizations.failureCode, NATIVE_RETAINED_OWNER_WAIT), eq(nativeRunFinalizations.attempt, 0), - isNull(nativeRunFinalizations.resultId), isNull(nativeRunFinalizations.leaseOwner), - sql`${nativeRunFinalizations.failureDetail}->'waitingForOwner'->>'runId' = ${owner.runId}`, - sql`${nativeRunFinalizations.failureDetail}->'waitingForOwner'->>'controllerBootId' = ${owner.bootId}`, - eq(heartbeatRuns.companyId, owner.companyId), eq(heartbeatRuns.nativeIssueId, owner.issueId), - eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.status, "running"), - isNull(heartbeatRuns.processPid), isNull(heartbeatRuns.processGroupId), isNull(heartbeatRuns.processStartedAt), - sql`${heartbeatRuns.nativeSessionId} = (select source.native_session_id from heartbeat_runs source - where source.id = ${owner.runId} and source.company_id = ${owner.companyId})`, +class SuccessorCoordinatorCreated extends Error {} + +async function lockSuccessorRows(tx: CheckpointTransaction, owner: NativeRunnerCheckpointFenceOwner, successorRunId: string) { + const coordinatorRow = (db: CheckpointTransaction) => db.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations) + .where(and(eq(nativeRunFinalizations.runId, successorRunId), eq(nativeRunFinalizations.companyId, owner.companyId))); + for (let pass = 0; ; pass++) { + try { + return await tx.transaction(async locking => { + const [coordinator] = await coordinatorRow(locking).for("share"); + const [run] = await locking.select({ runId: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, successorRunId), eq(heartbeatRuns.companyId, owner.companyId))).for("share"); + if (run && !coordinator && pass < 2 && (await coordinatorRow(locking)).length) throw new SuccessorCoordinatorCreated(); + return { coordinator, run }; + }); + } catch (error) { + if (!(error instanceof SuccessorCoordinatorCreated)) throw error; + } + } +} + +async function unstartedSuccessorOfOwner(tx: CheckpointTransaction, owner: NativeRunnerCheckpointFenceOwner, successorRunId: string) { + const { coordinator, run } = await lockSuccessorRows(tx, owner, successorRunId); + if (!run) return []; + const resumeClaim = sql`${nativeRunFinalizations.recoveryHistory}->-1`; + const waitsOnlyForOwner = (waiting: SQL) => sql`(coalesce(${waiting}, 'null'::jsonb) = 'null'::jsonb + or (${waiting}->>'runId' = ${owner.runId} and ${waiting}->>'controllerBootId' = ${owner.bootId}))`; + const noProcess = and(isNull(heartbeatRuns.processPid), isNull(heartbeatRuns.processGroupId), isNull(heartbeatRuns.processStartedAt)); + const preparing = and(isNull(nativeRunFinalizations.runId), eq(heartbeatRuns.runtimeMode, "legacy"), + eq(heartbeatRuns.issueId, owner.issueId), eq(heartbeatRuns.executionStage, "preparing"), isNotNull(heartbeatRuns.controllerBootId), + gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`), noProcess); + const admitted = and(eq(nativeRunFinalizations.companyId, owner.companyId), eq(nativeRunFinalizations.issueId, owner.issueId), + eq(nativeRunFinalizations.attempt, 0), isNull(nativeRunFinalizations.resultId), + eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.nativeIssueId, owner.issueId), + sql`${heartbeatRuns.nativeSessionId} = (select source.native_session_id from heartbeat_runs source + where source.id = ${owner.runId} and source.company_id = ${owner.companyId})`, + or(noProcess, sql`${heartbeatRuns.runnerProfileJson}->>'inheritedProcessIdentityRunId' = ${owner.runId}`), + or( + and(eq(nativeRunFinalizations.phase, "observed"), isNull(nativeRunFinalizations.leaseOwner), + isNull(nativeRunFinalizations.leaseExpiresAt), isNull(nativeRunFinalizations.failureCode), + sql`jsonb_array_length(${nativeRunFinalizations.recoveryHistory}) = 0`), + and(eq(nativeRunFinalizations.phase, "retryable_failure"), + eq(nativeRunFinalizations.failureCode, NATIVE_RETAINED_OWNER_WAIT), isNull(nativeRunFinalizations.leaseOwner), + waitsOnlyForOwner(sql`${nativeRunFinalizations.failureDetail}->'waitingForOwner'`)), + and(eq(nativeRunFinalizations.phase, "observed"), isNotNull(nativeRunFinalizations.leaseOwner), + gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`), + sql`${resumeClaim}->>'disposition' = ${SAME_RUN_RESUME_DISPOSITION}`, + sql`${resumeClaim}->>'controllerBootId' = ${nativeRunFinalizations.controllerBootId}`, + waitsOnlyForOwner(sql`${resumeClaim}->'evidence'->'waitingForOwner'`)), + )); + return tx.select({ runId: heartbeatRuns.id }).from(heartbeatRuns) + .leftJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id)).where(and( + eq(heartbeatRuns.id, successorRunId), eq(heartbeatRuns.companyId, owner.companyId), eq(heartbeatRuns.status, "running"), notExists(tx.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints).where(and( eq(nativeRunnerCheckpoints.companyId, owner.companyId), eq(nativeRunnerCheckpoints.runId, successorRunId)))), notExists(tx.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( eq(heartbeatRunEvents.companyId, owner.companyId), eq(heartbeatRunEvents.runId, successorRunId), isNull(heartbeatRunEvents.sourceEventId), inArray(heartbeatRunEvents.eventType, [...OWN_PROCESS_START_EVENT_TYPES])))), - )).for("share", { of: nativeRunFinalizations }); + coordinator ? admitted : preparing, + )); } export async function assertNativeRunnerCheckpointOwner(tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner, newWork: boolean) { diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 3083c864618..cadd37306f1 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -1386,7 +1386,7 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); await fixture.db.update(issues).set({ executionRunId: nextRunId }).where(eq(issues.id, fixture.issueId)); const before = (await loadNativeRunnerCheckpoint(fixture.db, binding))!; authority.queueCommand(scenario === "new_work" || scenario === "queued_new_work" ? "turn.start" : command, {}, `early-retirement-${scenario}`); - if (!["idle", "preparing_successor", "claimed_successor", "queued_successor", "scheduled_retry_successor"].includes(scenario)) { + if (!["idle", "preparing_successor", "foreign_preparation", "claimed_successor", "queued_successor", "scheduled_retry_successor"].includes(scenario)) { await expect(authority.flushCheckpoint()).rejects.toThrow("native_runner_checkpoint_owner_lost"); expect((await loadNativeRunnerCheckpoint(fixture.db, binding))?.records).toEqual(before.records); } else { diff --git a/server/src/services/native-runtime/same-run-resume-disposition.ts b/server/src/services/native-runtime/same-run-resume-disposition.ts new file mode 100644 index 00000000000..6e612e1deb0 --- /dev/null +++ b/server/src/services/native-runtime/same-run-resume-disposition.ts @@ -0,0 +1 @@ +export const SAME_RUN_RESUME_DISPOSITION = "same_run_resume"; diff --git a/server/src/services/native-runtime/scheduled-resume-continuation.ts b/server/src/services/native-runtime/scheduled-resume-continuation.ts index 94d40f37293..d98fdaf718e 100644 --- a/server/src/services/native-runtime/scheduled-resume-continuation.ts +++ b/server/src/services/native-runtime/scheduled-resume-continuation.ts @@ -3,8 +3,7 @@ import { and, eq } from "drizzle-orm"; import { heartbeatRuns } from "@paperclipai/db"; import { readNativeLocalProcessStop, readUnidentifiedLocalRunnerStop, recordUnidentifiedLocalRunnerStop } from "../native-local-process-stop.js"; import { NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT } from "./native-restart-recovery.js"; - -export const SAME_RUN_RESUME_DISPOSITION = "same_run_resume"; +import { SAME_RUN_RESUME_DISPOSITION } from "./same-run-resume-disposition.js"; export function sameRunResumeClaimEntry(input: { at: Date; controllerBootId: string; providerAttempt: number; failureDetail: Record | null | undefined; @@ -22,6 +21,7 @@ export function sameRunResumeClaimEntry(input: { checkpointExists: detail.checkpointExists ?? null, successorRunId: detail.successorRunId ?? null, replacementDenied: detail.replacementDenied ?? null, + waitingForOwner: detail.waitingForOwner ?? null, }, }; }