fix(native): name the missing precondition when a bootstrap restart refuses an unopened provider (#22406)

Adds a closed diagnostic suffix to the provider_identity_incomplete refusal raised when a same-run bootstrap restart meets an unopened provider: bootstrap_restart=<same_run|other_run>,provider=<unopened|opened>,proof=<held|not_recorded|stop outcome>. Recovery behaviour is unchanged.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-11-20 05:14:31 -08:00
1 parent bbb6b4fd5c
commit c3260b1c03
2 files changed
+46 -3

No files matched your search

@@ -22844,6 +22844,7 @@ describe("runnerd provider runtime wiring", () => {
{ outcome: "created", scenario: "interrupted_bootstrap_unopened", homeMode: "isolated" },
{ outcome: "created", scenario: "interrupted_bootstrap_unopened_active_turn", homeMode: "isolated" },
{ outcome: "created", scenario: "interrupted_bootstrap_unopened_unproven", homeMode: "isolated" },
{ outcome: "created", scenario: "interrupted_bootstrap_unopened_stop_unproven", homeMode: "isolated" },
])(
"bootstraps only an untouched provider session in a $outcome workspace lease: $scenario ($homeMode)", async ({ outcome, scenario, homeMode }) => {
const remoteCwd = join(isolatedStateDirectory, "remote");
@@ -23062,6 +23063,15 @@ describe("runnerd provider runtime wiring", () => {
"interrupted_bootstrap_unprepared_backup"].includes(scenario)) {
noteRemoteBootstrapRestartProven(execution.binding.runId, "stopped");
}
if (scenario === "interrupted_bootstrap_unopened_stop_unproven") {
expect(recordRemoteBootstrapRestartProof(execution.binding.runId, "stop_unproven")).toBe(false);
}
const refusedRestart = {
interrupted_bootstrap_unopened_unproven: "same_run,provider=unopened,proof=not_recorded",
interrupted_bootstrap_unopened_stop_unproven: "same_run,provider=unopened,proof=stop_unproven",
interrupted_bootstrap_unopened_other_run: "other_run,provider=unopened,proof=held",
interrupted_bootstrap_unopened_active_turn: "same_run,provider=opened,proof=held",
}[scenario];
if (scenario === "interrupted_bootstrap" || scenario === "interrupted_bootstrap_unopened" || scenario === "interrupted_bootstrap_unprepared"
|| scenario === "interrupted_bootstrap_unprepared_unrecorded" || scenario === "interrupted_bootstrap_connected_checkpoint_proven") {
await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined();
@@ -23125,7 +23135,9 @@ describe("runnerd provider runtime wiring", () => {
await expect(options.prepareExternalRunnerState()).rejects.toThrow(
outcome === "replacement" ? "runner_harness_backup_unavailable"
: scenario.startsWith("interrupted_bootstrap_unprepared") ? "runner_harness_state_mismatch: resumed_unavailable"
: "runner_harness_state_mismatch",
: refusedRestart
? `runner_harness_state_mismatch: provider_identity_incomplete (identity_binding; bootstrap_restart=${refusedRestart})`
: "runner_harness_state_mismatch",
);
expect(remoteExecute.mock.calls.some(([command]) => command.args?.[1]?.includes("install -d"))).toBe(false);
expect(syncIn).not.toHaveBeenCalled();
@@ -7197,9 +7197,19 @@ export function noteRemoteBootstrapRestartProven(runId: string, stopped: "absent
provenRemoteBootstrapRestarts.set(runId, stopped);
}
type RemoteBootstrapRestartProofOutcome = Awaited<ReturnType<typeof stopStaleRemoteRunnerBeforeBootstrap>>;
const REMOTE_BOOTSTRAP_RESTART_PROOF_OUTCOME_LIMIT = 1_000;
const remoteBootstrapRestartProofOutcomes = new Map<string, RemoteBootstrapRestartProofOutcome>();
export function recordRemoteBootstrapRestartProof(runId: string,
stopped: Awaited<ReturnType<typeof stopStaleRemoteRunnerBeforeBootstrap>>): boolean {
stopped: RemoteBootstrapRestartProofOutcome): boolean {
provenRemoteBootstrapRestarts.delete(runId);
remoteBootstrapRestartProofOutcomes.delete(runId);
remoteBootstrapRestartProofOutcomes.set(runId, stopped);
if (remoteBootstrapRestartProofOutcomes.size > REMOTE_BOOTSTRAP_RESTART_PROOF_OUTCOME_LIMIT) {
const oldest = remoteBootstrapRestartProofOutcomes.keys().next().value;
if (oldest !== undefined) remoteBootstrapRestartProofOutcomes.delete(oldest);
}
if (stopped !== "absent_confirmed" && stopped !== "stopped" && stopped !== "killed") return false;
provenRemoteBootstrapRestarts.set(runId, stopped);
return true;
@@ -7241,6 +7251,21 @@ export function takeRemoteBootstrapRestartProof(runId: string): boolean {
return provenRemoteBootstrapRestarts.delete(runId);
}
export function remoteBootstrapRestartRefusalDiagnostic(input: {
restartKind: string | null;
sameRun: boolean;
providerUnopened: boolean;
runId: string;
}): string {
if (input.restartKind !== "bootstrap_incomplete") return "";
const proof = hasRemoteBootstrapRestartProof(input.runId)
? "held"
: remoteBootstrapRestartProofOutcomes.get(input.runId) ?? "not_recorded";
remoteBootstrapRestartProofOutcomes.delete(input.runId);
return `; bootstrap_restart=${input.sameRun ? "same_run" : "other_run"},`
+ `provider=${input.providerUnopened ? "unopened" : "opened"},proof=${proof}`;
}
export async function stopStaleRemoteRunnerBeforeBootstrap(input: {
execution: NativeExecutionInput;
runnerExecutionTarget?: AdapterExecutionTarget | null;
@@ -17676,7 +17701,13 @@ async function createRunnerdBackendWithinSessionClaim(
notSuspendedRunId: input.execution.binding.runId,
};
}
throw new Error(`runner_harness_state_mismatch: provider_identity_incomplete (${durableProviderCheckpointFailureReason(input.execution, providerState)})`);
const restartRefusal = source === "host_preparation" ? remoteBootstrapRestartRefusalDiagnostic({
restartKind: input.restartRecovery?.kind ?? null,
sameRun: runnerState.runId === input.execution.binding.runId,
providerUnopened: remoteProviderStateNeverOpened(input.execution, providerState),
runId: input.execution.binding.runId,
}) : "";
throw new Error(`runner_harness_state_mismatch: provider_identity_incomplete (${durableProviderCheckpointFailureReason(input.execution, providerState)}${restartRefusal})`);
}
const previousManifest = compatibleNativeHarnessBackupManifests({
root,