diff --git a/server/src/services/chat-channels/failed-run-retry.ts b/server/src/services/chat-channels/failed-run-retry.ts index d90c7a6c989..533f013345c 100644 --- a/server/src/services/chat-channels/failed-run-retry.ts +++ b/server/src/services/chat-channels/failed-run-retry.ts @@ -566,6 +566,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti "provider_transport_failed", "native_runner_process_exited", "provider_session_open_rejected", + "local_runtime_api_listener_unavailable", ]; if ( !coordinator || diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 6013a88ba67..ac41ebd8d80 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -7310,6 +7310,7 @@ export function nativeSessionFailureSourceCode( | "provider_session_open_rejected" | "runner_ingress_access_rejected" | "runner_ingress_acquisition_failed" + | "local_runtime_api_listener_unavailable" | "native_session_interrupted" { if (error instanceof NativePermissionDeclinedError) return "native_permission_declined"; if (error instanceof NativeProviderStateIncompatibleError) return NATIVE_PROVIDER_STATE_INCOMPATIBLE; @@ -7410,6 +7411,12 @@ export function nativeSessionFailureSourceCode( } return "provider_session_open_rejected"; } + // This process has no HTTP listener (graceful shutdown closes it first), so + // the control plane cannot hand a runner its callback origin. The lookup runs + // before any runner or provider process is created. + if (/\blocal_runtime_api_listener_unavailable\b/.test(message)) { + return "local_runtime_api_listener_unavailable"; + } return "native_session_interrupted"; } @@ -7505,6 +7512,7 @@ export async function nativeProviderRecoveryEvidence(input: { "provider_request_protocol_error", "native_runner_process_exited", "provider_session_open_rejected", + "local_runtime_api_listener_unavailable", ]).has(input.sourceFailureCode); if (!checkpointExists && !providerEventsExist && (definitelyPreSession || unclaimedBootstrap && input.bootstrap?.transientIngressFailure)) { diff --git a/server/src/services/native-runtime/native-startup-retry.integration.test.ts b/server/src/services/native-runtime/native-startup-retry.integration.test.ts index 117280af19d..e847c5d27f8 100644 --- a/server/src/services/native-runtime/native-startup-retry.integration.test.ts +++ b/server/src/services/native-runtime/native-startup-retry.integration.test.ts @@ -40,6 +40,7 @@ import { environmentRunnerIngressErrorData, JsonRpcCallError, PluginEnvironmentR import { accountRunCost } from "../run-cost-accounting.js"; import { settleUnrecoverableExecutions } from "../execution-recovery-resolution.js"; import { agentProviderLimitCooldown } from "./native-startup-retry.js"; +import { resolveLocalRuntimeApiUrl, runtimeApiInternals } from "../../runtime-api.js"; type SessionOpenStatus = "failed" | "rejected" | "indeterminate"; @@ -479,6 +480,82 @@ describe("native provider startup retry", () => { expect(state.issue.status).toBe("in_progress"); }); + it("retries startup that a draining server interrupted before any provider session", async () => { + const seeded = await seedRun(await createAgent()); + // A graceful shutdown closes the HTTP listener first; startup that reaches + // the runtime API lookup afterwards fails before a runner or provider exists. + runtimeApiInternals.resetForTests(); + provider.execute.mockReset().mockImplementation(async () => { + resolveLocalRuntimeApiUrl(); + throw new Error("expected the closed listener to stop startup"); + }); + const failedAt = new Date(); + await expect(executePaperclipNativeSession({ + db, + execution: seeded.execution, + runnerInstanceId: seeded.runnerInstanceId, + backend: { + descriptor: async () => { throw new Error("Unexpected provider descriptor call"); }, + openSession: async () => { throw new Error("Unexpected provider session call"); }, + }, + })).rejects.toThrow("local_runtime_api_listener_unavailable"); + + const state = await read(seeded); + expect(state.coordinator).toMatchObject({ + phase: "retryable_failure", + recoveryState: "resuming_session", + attempt: 1, + leaseOwner: null, + failureDetail: expect.objectContaining({ + originalFailureCode: "local_runtime_api_listener_unavailable", + recoveryMode: "bootstrap_retry", + providerSessionEstablished: false, + providerEventsExist: false, + checkpointExists: false, + }), + }); + const [low, high] = backoffWindow(failedAt, 1); + expect(state.coordinator.nextAttemptAt!.getTime()).toBeGreaterThanOrEqual(low); + expect(state.coordinator.nextAttemptAt!.getTime()).toBeLessThanOrEqual(high + 5_000); + expect(state.issue.status).toBe("in_progress"); + expect(state.action).toMatchObject({ + ownerType: "agent", + ownerAgentId: seeded.agentId, + wakePolicy: { kind: "resume_native_run", runId: seeded.runId }, + }); + // The next server process claims the same run; no replacement run is created. + await claimRetry(seeded); + }); + + it("keeps a closed-listener failure operator-owned once provider work is durable", async () => { + const seeded = await seedRun(await createAgent()); + runtimeApiInternals.resetForTests(); + provider.execute.mockReset().mockImplementation(async () => { + await db.insert(heartbeatRunEvents).values({ + companyId, runId: seeded.runId, agentId: seeded.agentId, seq: 900, + eventType: "turn.started", stream: "system", + }); + resolveLocalRuntimeApiUrl(); + }); + await expect(executePaperclipNativeSession({ + db, + execution: seeded.execution, + runnerInstanceId: seeded.runnerInstanceId, + backend: { + descriptor: async () => { throw new Error("Unexpected provider descriptor call"); }, + openSession: async () => { throw new Error("Unexpected provider session call"); }, + }, + })).rejects.toThrow("local_runtime_api_listener_unavailable"); + + const state = await read(seeded); + expect(state.coordinator).toMatchObject({ + phase: "terminal_failure", + nextAttemptAt: null, + failureDetail: expect.objectContaining({ recoveryMode: "ambiguous_state", providerSessionEstablished: true }), + }); + expect(state.issue.status).toBe("blocked"); + }); + it.each(["grok_model_mismatch", "requested_model_unsupported"])( "stops at a session.open %s rejection with a choose-model action instead of a doomed retry", async (classification) => {