fix: preserve pre-provider recovery after verification outages

This commit is contained in:
Dotta committed 2026-10-10 23:05:40 -05:00
1 parent 0a0ccbaac1
commit ad27e0069c
2 files changed
+37

No files matched your search

@@ -1,5 +1,6 @@
import { compareAndSealRetiredComputerCheckpoint, retiredComputerCheckpointIsSettled, retiredComputerCheckpointUnsettledReason, retireUnlaunchedComputerAttempt, nativeSessionBootstrapTimeoutMs } from "./native-session-executor.js";
import { prepareHeartbeatGitHubLaunchers } from "../heartbeat-github-launchers.js";
import { RemoteProviderPackVerificationError } from "./remote-provider-pack-cache.js";
import { gunzipSync } from "node:zlib";
import { configuredEnvironmentProjection } from "../../vendor/paperclip-runner/index.js";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
@@ -9743,6 +9744,37 @@ describe("native session bounded recovery", () => {
})).toMatchObject({ recoveryMode: mode });
});
it.each(["timeout", "transport_error", "command_failed"] as const)(
"keeps unavailable pack verification distinct and bounds startup retries (%s)", (reason) => {
const code = nativeSessionFailureSourceCode(new RemoteProviderPackVerificationError("unavailable", reason));
expect(code).toBe("runner_remote_provider_verification_unavailable");
const now = new Date("2026-10-11T00:00:00Z");
expect(nativeSessionFailureDisposition(1, now, code)).toEqual({
phase: "retryable_failure", failureCode: "native_session_interrupted",
nextAttemptAt: new Date("2026-10-11T00:00:30Z"),
});
expect(nativeSessionFailureDisposition(3, now, code)).toEqual({
phase: "terminal_failure", failureCode: "native_session_retry_exhausted", nextAttemptAt: null,
});
},
);
it.each([
[{}, [], "bootstrap_retry"],
[{ uncertain: true }, [], "ambiguous_state"],
[{}, [{ eventType: "turn.started" }], "ambiguous_state"],
[{ providerSessionId: "existing-provider" }, [], "exact_checkpoint_resume"],
[{ providerSessionId: "existing-provider", terminal: { runTerminalState: "failed" } }, [], "ambiguous_state"],
])("unavailable pack verification preserves provider recovery evidence (%j)", async (checkpoint, events, mode) => {
const code = nativeSessionFailureSourceCode(new RemoteProviderPackVerificationError("unavailable", "timeout"));
let reads = 0;
const db = { select: () => ({ from: () => ({ where: () => ({
limit: async () => ++reads === 1 ? [{ runnerProfileJson: { sessionCheckpoint: checkpoint } }] : events,
}) }) }) };
expect(await nativeProviderRecoveryEvidence({ db: db as never, runId: "run", sourceFailureCode: code }))
.toMatchObject({ recoveryMode: mode });
});
it("requires operator action without retrying an approval-required terminal", () => {
const code = nativeSessionFailureSourceCode(
new NativeProviderTerminalFailure("approval_required", false, "Approval required"),
@@ -6430,6 +6430,7 @@ export function nativeSessionFailureSourceCode(
| "native_session_cleanup_quarantined"
| "native_adopted_runner_authentication_timeout"
| "runner_remote_provider_artifact_incompatible"
| "runner_remote_provider_verification_unavailable"
| "runner_remote_artifact_platform_mismatch"
| "provider_process_exited"
| "provider_stdout_closed"
@@ -6477,6 +6478,9 @@ export function nativeSessionFailureSourceCode(
if (/runner_remote_provider_artifact_incompatible/i.test(message)) {
return "runner_remote_provider_artifact_incompatible";
}
if (/^runner_remote_provider_verification_unavailable(?::|$)/.test(message)) {
return "runner_remote_provider_verification_unavailable";
}
if (/provider_process_exited/i.test(message)) {
return "provider_process_exited";
}
@@ -6601,6 +6605,7 @@ export async function nativeProviderRecoveryEvidence(input: {
ReturnType<typeof nativeSessionFailureSourceCode>
>([
"runner_remote_provider_artifact_incompatible",
"runner_remote_provider_verification_unavailable",
"runner_remote_artifact_platform_mismatch",
"provider_process_exited",
"provider_stdout_closed",