diff --git a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts index b682b02f785..49cac6d9fbb 100644 --- a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts @@ -480,18 +480,23 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d loadCheckpoint: () => checkpoint, canReuseProcess: () => true, ...overrides, }); const denied = "runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate"; - await expect(inspect({ execution: { ...current, binding: { ...current.binding, companyId: randomUUID() } } })) - .rejects.toThrow(denied); - await expect(inspect({ matchesExecution: () => false })).rejects.toThrow(denied); + const ordinary = async (attempt: Promise) => { + const error = await attempt.then(() => null, failure => failure); + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain(denied); + expect(error).not.toBeInstanceOf(NativePredecessorRetirementUnprovenError); + }; + await ordinary(inspect({ execution: { ...current, binding: { ...current.binding, companyId: randomUUID() } } })); + await ordinary(inspect({ matchesExecution: () => false })); await expect(readSandbox({ remoteCwd: "/unrelated" })).rejects.toThrow("retained_sandbox_binding"); for (const leaseExpiresAt of [null, new Date(Date.now() - 1_000)]) { await db.update(nativeRunFinalizations).set({ leaseExpiresAt }).where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); - await expect(inspect()).rejects.toThrow(denied); + await ordinary(inspect()); await expect(readSandbox()).rejects.toThrow("retained_sandbox_binding"); } await db.update(nativeRunFinalizations).set({ leaseExpiresAt: deadline }).where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, fixture.scope.runId)); - await expect(inspect()).rejects.toThrow(denied); + await ordinary(inspect()); await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, fixture.scope.runId)); await expect(readSandbox()).rejects.toMatchObject({ retryAt: deadline }); const waiting = await inspect().catch(error => error); @@ -505,7 +510,7 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d .where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); await expect(readSandbox()).resolves.toEqual({ remoteCwd: "/workspace" }); await expect(inspect({ readControlState: () => ({ identity, commands: [{ status: "pending" }], - committedEvents: [{ eventType: "run.terminal", envelope: identity }] }) })).rejects.toThrow(denied); + committedEvents: [{ eventType: "run.terminal", envelope: identity }] }) })).rejects.toBeInstanceOf(NativePredecessorRetirementUnprovenError); // A created or replacement sandbox holds no Runner to adopt. Live adoption // steps aside so ordinary admission can retire the finished owner, instead // of denying with prior_owner_terminal_state_indeterminate (Nov 10 staging). @@ -582,6 +587,15 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d fallback(await inspect(() => ({ ...parked, commands: [...parked.commands, { type: "runner.suspend", status: "pending" }] })) .catch(error => error), "journal_unsettled"); + for (const status of ["failed", "running"] as const) { + await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, fixture.scope.runId)); + const unsettled = await inspect(() => ({ ...parked, commands: [...parked.commands, { type: "runner.suspend", status: "pending" }] })) + .catch(error => error); + expect(unsettled).not.toBeInstanceOf(NativePredecessorRetirementUnprovenError); + expect(unsettled.message).toBe("runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate; warm_recovery_refused=owner_unsettled"); + } + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, fixture.scope.runId)); + const recovered = (await inspect())!; await expect(recovered.revalidate("replacement-controller")).resolves.toBeUndefined(); await db.update(nativeRunFinalizations).set({ leaseOwner: "native-finalizer:replacement", leaseExpiresAt: new Date(Date.now() + 60_000) }) diff --git a/server/src/services/remote-workspace-unchanged-proof.ts b/server/src/services/remote-workspace-unchanged-proof.ts index 7c0fd82a5d1..67477468de5 100644 --- a/server/src/services/remote-workspace-unchanged-proof.ts +++ b/server/src/services/remote-workspace-unchanged-proof.ts @@ -7,7 +7,6 @@ import { remoteWorkspaceRunReferenceSchema, type RemoteWorkspaceAuthority } from import { executionStartedEventPredicate } from "./pre-dispatch-execution-evidence.js"; import { runOwnerFenceHeldSql } from "./remote-workspace-writer-hold.js"; import { carriesInheritedRunnerIdentity } from "./native-runtime/native-inherited-runner-retry.js"; -import { PROCESS_IDENTITY_RECORDED } from "./native-local-process-stop.js"; /** * A writer's prepared files equal its saved version only while nothing ran in @@ -46,15 +45,6 @@ const FILE_NEUTRAL_OPERATION_PHASES = ["remote_workspace_restore", "workspace_fi /** Terminal states of a run that ended without a result. */ const ENDED_WITHOUT_RESULT = ["failed", "cancelled", "interrupted", "timed_out"]; -async function carriesOnlyInheritedRunnerIdentity(tx: Pick, - run: Parameters[1]): Promise { - if (!await carriesInheritedRunnerIdentity(tx, run)) return false; - const [own] = await tx.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( - eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), - eq(heartbeatRunEvents.eventType, PROCESS_IDENTITY_RECORDED))).limit(1); - return !own; -} - export type UnchangedProofRefusal = | "baseline_missing" | "authority_mismatch" | "writer_not_ended" | "writer_owner_live" | "process_recorded" | "reference_mismatch" | "provider_session_started" | "execution_recorded" | "result_recorded" @@ -90,7 +80,7 @@ export async function proveRemoteWorkspaceUnchanged(tx: Pick, inpu if (!input.callerOwnsWriter && run.ownerFenceHeld) return { proven: false, reason: "writer_owner_live" }; if (run.sessionIdAfter !== null) return { proven: false, reason: "process_recorded" }; if ((run.processPid !== null || run.processGroupId !== null || run.processStartedAt !== null) - && !await carriesOnlyInheritedRunnerIdentity(tx, run)) return { proven: false, reason: "process_recorded" }; + && !await carriesInheritedRunnerIdentity(tx, run)) return { proven: false, reason: "process_recorded" }; const profile = object(run.runnerProfileJson); const reference = remoteWorkspaceRunReferenceSchema.safeParse(profile.remoteWorkspace); if (profile.nativeWorkspaceSync != null || !reference.success || reference.data.state !== "prepared"