diff --git a/server/src/__tests__/native-session-resumption.test.ts b/server/src/__tests__/native-session-resumption.test.ts index 2c6873dc213..db322b28ef5 100644 --- a/server/src/__tests__/native-session-resumption.test.ts +++ b/server/src/__tests__/native-session-resumption.test.ts @@ -1,4 +1,5 @@ import { readFile } from "node:fs/promises"; +import { createRunLogWriterAuthority } from "../services/heartbeat/run-log-authority.js"; import { enableRunFailureReporting } from "./helpers/run-failure-reporting.js"; import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { spawn } from "node:child_process"; @@ -402,6 +403,24 @@ describe("P6-25 pre-result native session recovery", () => { await expect(db.select().from(nativeRunFinalizations)).resolves.toHaveLength(8); }); + it("lets a resume claimed by a restarted process write the run's existing log (storage chaos ST2 t01)", async () => { + await db.update(nativeRunFinalizations).set({ + phase: "retryable_failure", leaseOwner: null, leaseExpiresAt: null, nextAttemptAt: new Date(0), + controllerBootId: "previous-process-boot", + }).where(eq(nativeRunFinalizations.runId, runId)); + await db.update(heartbeatRuns).set({ status: "failed", logStore: "s3_segments", logRef: `${companyId}/run-log` }) + .where(eq(heartbeatRuns.id, runId)); + const dispatched: Array<{ runId: string; leaseOwner: string; attempt: number }> = []; + await dispatchNativeSessionResumptions({ db, runnerInstanceId: "heartbeat-restarted", runIds: [runId], + dispatch: (claim) => dispatched.push(claim) }); + expect(dispatched).toHaveLength(1); + const [resumed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + await expect(createRunLogWriterAuthority(db, resumed!, dispatched[0]!.leaseOwner)()).resolves.toBeUndefined(); + const [owner] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(owner).toMatchObject({ controllerPid: process.pid, leaseOwner: dispatched[0]!.leaseOwner }); + expect(owner!.controllerBootId).not.toBe("previous-process-boot"); + }); + it("uses provider-neutral checkpoint-free bootstrap retry only when durable evidence proves no session existed", async () => { await expect(claimNativeSessionResumptions({ db, diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index d915c90cc55..97782172df6 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -1,4 +1,5 @@ import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; +import { currentControllerProcessStamp } from "../controller-process-identity.js"; import { NATIVE_SESSION_RESUMPTION_ISSUE_STATUSES } from "./native-session-resumption-statuses.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { pendingNativeRunProjectionCondition } from "./native-run-projection.js"; @@ -312,6 +313,11 @@ export async function claimNativeSessionResumptions(input: { phase: "observed", leaseOwner, leaseExpiresAt, + ...(() => { + const stamp = currentControllerProcessStamp(); + return { controllerBootId: stamp.bootId, controllerPid: stamp.pid, controllerHostHash: stamp.hostHash, + controllerProcessStartedAt: stamp.processStartedAt ? new Date(stamp.processStartedAt) : null }; + })(), failureCode: null, failureDetail: null, nextAttemptAt: null,