From ad27e0069cff4518ce94f0cfb15392dfe466ab48 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 10 Oct 2026 23:04:31 -0500 Subject: [PATCH] fix: preserve pre-provider recovery after verification outages --- .../native-session-executor.test.ts | 32 +++++++++++++++++++ .../native-runtime/native-session-executor.ts | 5 +++ 2 files changed, 37 insertions(+) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 5203daf75cc..ec8617a5515 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1,5 +1,6 @@ import { compareAndSealRetiredComputerCheckpoint, retiredComputerCheckpointIsSettled, retiredComputerCheckpointUnsettledReason, retireUnlaunchedComputerAttempt, nativeSessionBootstrapTimeoutMs } from "./native-session-executor.js"; import { prepareHeartbeatGitHubLaunchers } from "../heartbeat-github-launchers.js"; +import { RemoteProviderPackVerificationError } from "./remote-provider-pack-cache.js"; import { gunzipSync } from "node:zlib"; import { configuredEnvironmentProjection } from "../../vendor/paperclip-runner/index.js"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -9743,6 +9744,37 @@ describe("native session bounded recovery", () => { })).toMatchObject({ recoveryMode: mode }); }); + it.each(["timeout", "transport_error", "command_failed"] as const)( + "keeps unavailable pack verification distinct and bounds startup retries (%s)", (reason) => { + const code = nativeSessionFailureSourceCode(new RemoteProviderPackVerificationError("unavailable", reason)); + expect(code).toBe("runner_remote_provider_verification_unavailable"); + const now = new Date("2026-10-11T00:00:00Z"); + expect(nativeSessionFailureDisposition(1, now, code)).toEqual({ + phase: "retryable_failure", failureCode: "native_session_interrupted", + nextAttemptAt: new Date("2026-10-11T00:00:30Z"), + }); + expect(nativeSessionFailureDisposition(3, now, code)).toEqual({ + phase: "terminal_failure", failureCode: "native_session_retry_exhausted", nextAttemptAt: null, + }); + }, + ); + + it.each([ + [{}, [], "bootstrap_retry"], + [{ uncertain: true }, [], "ambiguous_state"], + [{}, [{ eventType: "turn.started" }], "ambiguous_state"], + [{ providerSessionId: "existing-provider" }, [], "exact_checkpoint_resume"], + [{ providerSessionId: "existing-provider", terminal: { runTerminalState: "failed" } }, [], "ambiguous_state"], + ])("unavailable pack verification preserves provider recovery evidence (%j)", async (checkpoint, events, mode) => { + const code = nativeSessionFailureSourceCode(new RemoteProviderPackVerificationError("unavailable", "timeout")); + let reads = 0; + const db = { select: () => ({ from: () => ({ where: () => ({ + limit: async () => ++reads === 1 ? [{ runnerProfileJson: { sessionCheckpoint: checkpoint } }] : events, + }) }) }) }; + expect(await nativeProviderRecoveryEvidence({ db: db as never, runId: "run", sourceFailureCode: code })) + .toMatchObject({ recoveryMode: mode }); + }); + it("requires operator action without retrying an approval-required terminal", () => { const code = nativeSessionFailureSourceCode( new NativeProviderTerminalFailure("approval_required", false, "Approval required"), diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index b4130a56c79..e6463f11a94 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -6430,6 +6430,7 @@ export function nativeSessionFailureSourceCode( | "native_session_cleanup_quarantined" | "native_adopted_runner_authentication_timeout" | "runner_remote_provider_artifact_incompatible" + | "runner_remote_provider_verification_unavailable" | "runner_remote_artifact_platform_mismatch" | "provider_process_exited" | "provider_stdout_closed" @@ -6477,6 +6478,9 @@ export function nativeSessionFailureSourceCode( if (/runner_remote_provider_artifact_incompatible/i.test(message)) { return "runner_remote_provider_artifact_incompatible"; } + if (/^runner_remote_provider_verification_unavailable(?::|$)/.test(message)) { + return "runner_remote_provider_verification_unavailable"; + } if (/provider_process_exited/i.test(message)) { return "provider_process_exited"; } @@ -6601,6 +6605,7 @@ export async function nativeProviderRecoveryEvidence(input: { ReturnType >([ "runner_remote_provider_artifact_incompatible", + "runner_remote_provider_verification_unavailable", "runner_remote_artifact_platform_mismatch", "provider_process_exited", "provider_stdout_closed",