diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index c1b67f1737f..d96451efb43 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -13556,6 +13556,101 @@ describe("native session bounded recovery", () => { expect(state.upsertRecoveryAction).not.toHaveBeenCalled(); }); + // H1 (restart chaos): a budget pause's lifecycle Stop interrupted a Codex + // turn, which ended without a semantic result. That Stop must stay a + // cancellation (resume continues it), never an interrupted-run retry the + // pause then strands. + it.each(["Agent lifecycle stop requested", "Cancelled due to budget pause"])("does not turn an acknowledged governance pause Stop (%s) into a retry", async (reason) => { + const updates: Array<{ table: unknown; values: Record }> = []; + const stop: Record = {}; + state.execute.mockReset().mockImplementationOnce(async () => { + Object.assign(stop, { + cancellation: { reason, source: "control_plane", expected: true, initiator: { type: "system" }, + recordedAt: new Date().toISOString() }, + ...(reason === "Agent lifecycle stop requested" ? { pauseStop: { holds: ["company_paused"], companyPauseReason: "budget" } } : {}), + nativeCancellation: { + schema: "paperclip.native-cancellation.v1", ...execution.binding, scope: "run", reasonCode: "cancellation_run_only", + dispatched: true, dispatchState: "acknowledged", intentAuditId: "intent", acknowledgementAuditId: "ack", + } }); + throw new Error("native_finalization_missing: session returned no semantic result"); + }); + state.upsertRecoveryAction.mockClear(); + await expect(executePaperclipNativeSession({ + db: leaseDb(execution, {}, stop, updates), execution, runnerInstanceId: "governance-stop", + })).rejects.toThrow("native_cancellation_pending_recovery"); + expect(updates.some(update => update.table === heartbeatRuns && update.values.status === "failed")).toBe(false); + expect(updates.some(update => update.table === nativeRunFinalizations && update.values.failureCode === "native_retry_cancelled")).toBe(true); + expect(state.upsertRecoveryAction).not.toHaveBeenCalled(); + }); + + it("waits for an in-flight governance pause Stop to save its acknowledgement before judging a missing result", async () => { + let releaseCleanup!: () => void, endTurn!: () => void, opened!: () => void; + const turnEnded = new Promise(resolve => { endTurn = resolve; }); + const sessionOpen = new Promise(resolve => { opened = resolve; }); + state.cancel.mockReset().mockReturnValue({ cleanup: new Promise(resolve => { releaseCleanup = resolve; }) }); + state.execute.mockReset().mockImplementationOnce(async (options) => { + await options.onSession({ cancel: state.cancel }); + opened(); + await turnEnded; + throw new Error("native_finalization_missing: session returned no semantic result"); + }); + state.upsertRecoveryAction.mockClear(); + const runResultJson: Record = {}; + const updates: Array<{ table: unknown; values: Record }> = []; + const outcome = executePaperclipNativeSession({ db: leaseDb(execution, {}, runResultJson, updates), execution, runnerInstanceId: "pause-stop-in-flight" }) + .catch(error => error); + // The pause's fence recorded its cancellation before dispatch. + const persistence = cancellationDb({ resultJson: { + cancellation: { reason: "Agent lifecycle stop requested", source: "control_plane", expected: true, + initiator: { type: "system" }, recordedAt: new Date().toISOString() }, + pauseStop: { holds: ["company_paused"], companyPauseReason: "budget" } } }); + const transaction = persistence.db.transaction.bind(persistence.db); + // The executor reads the same durable run row the Stop writes. + (persistence.db as unknown as { transaction: typeof transaction }).transaction = (async (operation: Parameters[0]) => { + const result = await transaction(operation); + Object.assign(runResultJson, persistence.getResultJson()); + return result; + }) as typeof transaction; + await sessionOpen; + const stopping = cancelNativeSession(execution.binding.runId, "Agent lifecycle stop requested", { db: persistence.db, scope: "run" }); + try { + await vi.waitFor(() => expect(state.cancel).toHaveBeenCalledOnce()); + // The interrupted turn ends before the Stop saves its acknowledgement. + endTurn(); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(updates.some(update => update.table === heartbeatRuns && update.values.status === "failed")).toBe(false); + releaseCleanup(); + await stopping; + expect(await outcome).toMatchObject({ name: "NativeCancellationPendingRecoveryError" }); + expect(updates.some(update => update.table === heartbeatRuns && update.values.status === "failed")).toBe(false); + expect(updates.some(update => update.table === nativeRunFinalizations && update.values.failureCode === "native_retry_cancelled")).toBe(true); + expect(state.upsertRecoveryAction).not.toHaveBeenCalled(); + } finally { releaseCleanup(); endTurn(); await outcome; } + }); + + it("still retries a missing result after a system Stop that is not a governance pause", async () => { + const updates: Array<{ table: unknown; values: Record }> = []; + const stop: Record = {}; + state.execute.mockReset().mockImplementationOnce(async () => { + Object.assign(stop, { + cancellation: { reason: "Agent lifecycle stop requested", source: "control_plane", expected: true, + initiator: { type: "system" }, recordedAt: new Date().toISOString() }, + nativeCancellation: { + schema: "paperclip.native-cancellation.v1", ...execution.binding, scope: "run", reasonCode: "cancellation_run_only", + dispatched: true, dispatchState: "acknowledged", intentAuditId: "intent", acknowledgementAuditId: "ack", + } }); + throw new Error("native_finalization_missing: session returned no semantic result"); + }); + state.upsertRecoveryAction.mockReset().mockResolvedValue({}); + const service = vi.spyOn(issueServiceModule, "issueService").mockReturnValue({ update: vi.fn(async () => null) } as unknown as ReturnType); + try { + await expect(executePaperclipNativeSession({ + db: leaseDb(execution, {}, stop, updates), execution, runnerInstanceId: "termination-stop", + })).rejects.not.toThrow("native_cancellation_pending_recovery"); + expect(updates.some(update => update.table === nativeRunFinalizations && update.values.failureCode === "native_retry_cancelled")).toBe(false); + } finally { service.mockRestore(); } + }); + it.each(["pending", "acknowledged"])("preserves a %s Stop when cancellation wins before the first turn", async (dispatchState) => { const updates: Array<{ table: unknown; values: Record }> = []; const stop: Record = {}; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index ed4ffb7cae8..a908267d1c0 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,4 +1,5 @@ import { remoteRecoveryLeaseMismatch } from "./remote-recovery-lease-mismatch.js"; +import { isGovernancePauseStop } from "../governance-pause-stop.js"; import { nativeLeaseHolder, notifyNativeTokenLeaseClaimed } from "./native-token-lease-settlement.js"; import { captureRemoteNativeProviderArtifacts } from "./native-provider-artifact-capture.js"; import { NativeSessionDiagnostics, nativeSessionDiagnostics, type NativeDiagnosticLease } from "./native-session-diagnostics.js"; @@ -11581,9 +11582,14 @@ async function executePaperclipNativeSessionWithinScope( } trace.activate(taskSettleScope); liveQuestions.close(); + const activeAtFailure = activeNativeSessions.get(input.execution.binding.runId); activeNativeSessions.delete(input.execution.binding.runId); clearSteeringDeliveries(input.execution.binding.runId); clearNativeRuntimeRequestResolutions(input.execution.binding.runId); + // A Stop interrupts the provider before it saves its acknowledgement. + // When the interrupted turn ends first, wait for that Stop to settle, so + // the checks below judge its saved receipt, not a failure. + if (activeAtFailure?.cancelRequested) await activeAtFailure.cancellationSettled?.catch(() => undefined); // Stop before paperclip_finish is normal. Bounded provider teardown has // finished; a missing result must not overwrite cancellation or trigger // another turn to perform completion bookkeeping. @@ -11608,7 +11614,14 @@ async function executePaperclipNativeSessionWithinScope( stopIntent.scope === "run" && typeof stopIntent.intentAuditId === "string" && stopIntent.intentAuditId.length > 0 && ["pending", "acknowledged"].includes(String(stopIntent.dispatchState)); + // A governance pause (budget, company or agent pause) Stop is the same + // kind of intentional stop. The fence records its cancellation before + // dispatch, so judge it whether or not the terminal write has landed; + // resume continues it (budget-resume-continuation.ts). + const governancePauseStop = !!stoppedRun && hasAcknowledgedNativeRunStopReceipt(stoppedRun) + && isGovernancePauseStop({ status: "cancelled", resultJson: stoppedRun.resultJson }); if (stoppedRun && (pendingStartupStop || hasAcknowledgedNativeStopIntent(stoppedRun) || hasAcknowledgedNativeReassignmentStopIntent(stoppedRun) + || governancePauseStop || (error instanceof NativeCancellationPendingRecoveryError && hasAcknowledgedNativeRunStopReceipt(stoppedRun)))) { const [settled] = await writeExecutionControl(input.db, [DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.accounting], tx => tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_retry_cancelled", nextAttemptAt: null,