diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 29858a3d5f8..9df1d167895 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -22844,6 +22844,7 @@ describe("runnerd provider runtime wiring", () => { { outcome: "created", scenario: "interrupted_bootstrap_unopened", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_unopened_active_turn", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_unopened_unproven", homeMode: "isolated" }, + { outcome: "created", scenario: "interrupted_bootstrap_unopened_stop_unproven", homeMode: "isolated" }, ])( "bootstraps only an untouched provider session in a $outcome workspace lease: $scenario ($homeMode)", async ({ outcome, scenario, homeMode }) => { const remoteCwd = join(isolatedStateDirectory, "remote"); @@ -23062,6 +23063,15 @@ describe("runnerd provider runtime wiring", () => { "interrupted_bootstrap_unprepared_backup"].includes(scenario)) { noteRemoteBootstrapRestartProven(execution.binding.runId, "stopped"); } + if (scenario === "interrupted_bootstrap_unopened_stop_unproven") { + expect(recordRemoteBootstrapRestartProof(execution.binding.runId, "stop_unproven")).toBe(false); + } + const refusedRestart = { + interrupted_bootstrap_unopened_unproven: "same_run,provider=unopened,proof=not_recorded", + interrupted_bootstrap_unopened_stop_unproven: "same_run,provider=unopened,proof=stop_unproven", + interrupted_bootstrap_unopened_other_run: "other_run,provider=unopened,proof=held", + interrupted_bootstrap_unopened_active_turn: "same_run,provider=opened,proof=held", + }[scenario]; if (scenario === "interrupted_bootstrap" || scenario === "interrupted_bootstrap_unopened" || scenario === "interrupted_bootstrap_unprepared" || scenario === "interrupted_bootstrap_unprepared_unrecorded" || scenario === "interrupted_bootstrap_connected_checkpoint_proven") { await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); @@ -23125,7 +23135,9 @@ describe("runnerd provider runtime wiring", () => { await expect(options.prepareExternalRunnerState()).rejects.toThrow( outcome === "replacement" ? "runner_harness_backup_unavailable" : scenario.startsWith("interrupted_bootstrap_unprepared") ? "runner_harness_state_mismatch: resumed_unavailable" - : "runner_harness_state_mismatch", + : refusedRestart + ? `runner_harness_state_mismatch: provider_identity_incomplete (identity_binding; bootstrap_restart=${refusedRestart})` + : "runner_harness_state_mismatch", ); expect(remoteExecute.mock.calls.some(([command]) => command.args?.[1]?.includes("install -d"))).toBe(false); expect(syncIn).not.toHaveBeenCalled(); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 328ebbab345..ecba5b780de 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -7197,9 +7197,19 @@ export function noteRemoteBootstrapRestartProven(runId: string, stopped: "absent provenRemoteBootstrapRestarts.set(runId, stopped); } +type RemoteBootstrapRestartProofOutcome = Awaited>; +const REMOTE_BOOTSTRAP_RESTART_PROOF_OUTCOME_LIMIT = 1_000; +const remoteBootstrapRestartProofOutcomes = new Map(); + export function recordRemoteBootstrapRestartProof(runId: string, - stopped: Awaited>): boolean { + stopped: RemoteBootstrapRestartProofOutcome): boolean { provenRemoteBootstrapRestarts.delete(runId); + remoteBootstrapRestartProofOutcomes.delete(runId); + remoteBootstrapRestartProofOutcomes.set(runId, stopped); + if (remoteBootstrapRestartProofOutcomes.size > REMOTE_BOOTSTRAP_RESTART_PROOF_OUTCOME_LIMIT) { + const oldest = remoteBootstrapRestartProofOutcomes.keys().next().value; + if (oldest !== undefined) remoteBootstrapRestartProofOutcomes.delete(oldest); + } if (stopped !== "absent_confirmed" && stopped !== "stopped" && stopped !== "killed") return false; provenRemoteBootstrapRestarts.set(runId, stopped); return true; @@ -7241,6 +7251,21 @@ export function takeRemoteBootstrapRestartProof(runId: string): boolean { return provenRemoteBootstrapRestarts.delete(runId); } +export function remoteBootstrapRestartRefusalDiagnostic(input: { + restartKind: string | null; + sameRun: boolean; + providerUnopened: boolean; + runId: string; +}): string { + if (input.restartKind !== "bootstrap_incomplete") return ""; + const proof = hasRemoteBootstrapRestartProof(input.runId) + ? "held" + : remoteBootstrapRestartProofOutcomes.get(input.runId) ?? "not_recorded"; + remoteBootstrapRestartProofOutcomes.delete(input.runId); + return `; bootstrap_restart=${input.sameRun ? "same_run" : "other_run"},` + + `provider=${input.providerUnopened ? "unopened" : "opened"},proof=${proof}`; +} + export async function stopStaleRemoteRunnerBeforeBootstrap(input: { execution: NativeExecutionInput; runnerExecutionTarget?: AdapterExecutionTarget | null; @@ -17676,7 +17701,13 @@ async function createRunnerdBackendWithinSessionClaim( notSuspendedRunId: input.execution.binding.runId, }; } - throw new Error(`runner_harness_state_mismatch: provider_identity_incomplete (${durableProviderCheckpointFailureReason(input.execution, providerState)})`); + const restartRefusal = source === "host_preparation" ? remoteBootstrapRestartRefusalDiagnostic({ + restartKind: input.restartRecovery?.kind ?? null, + sameRun: runnerState.runId === input.execution.binding.runId, + providerUnopened: remoteProviderStateNeverOpened(input.execution, providerState), + runId: input.execution.binding.runId, + }) : ""; + throw new Error(`runner_harness_state_mismatch: provider_identity_incomplete (${durableProviderCheckpointFailureReason(input.execution, providerState)}${restartRefusal})`); } const previousManifest = compatibleNativeHarnessBackupManifests({ root,