diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index dd961a60dc2..29669b434ca 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -474,6 +474,7 @@ import { retryTransientRemoteCheckpointRead, verifyNativeHarnessBackup, shouldRestoreNativeHarnessBackupIntoSandbox, + remoteProviderStateNeverOpened, } from "./native-session-executor.js"; import { onNativeRunnerEvidence } from "./native-runner-evidence-signal.js"; @@ -20274,10 +20275,15 @@ describe("runnerd provider runtime wiring", () => { ["fresh", "external_checkpoint", "external_pending_turn", "external_wrong_identity", "external_unregistered", "existing_state", "symlink_parent", "wrong_identity", "wrong_runner", "wrong_lease", "missing_control_plane", "connected", "committed_event", "committed_command", "pending_turn", "remote_probe_failed", "backup_present"] .map((scenario) => ({ outcome, scenario, homeMode: "isolated" }))), ...["resumed", "replacement"].map(outcome => ({ outcome, scenario: "fresh", homeMode: "preconfigured" })), - ...["interrupted_bootstrap", "interrupted_bootstrap_other_run", "interrupted_bootstrap_unproven", "interrupted_bootstrap_symlink_parent"] + ...["interrupted_bootstrap", "interrupted_bootstrap_other_run", "interrupted_bootstrap_unproven", "interrupted_bootstrap_symlink_parent", + "interrupted_bootstrap_unopened", "interrupted_bootstrap_unopened_unproven", "interrupted_bootstrap_unopened_other_run", + "interrupted_bootstrap_unopened_active_turn"] .map(scenario => ({ outcome: "resumed", scenario, homeMode: "isolated" })), { outcome: "created", scenario: "interrupted_bootstrap", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_symlink_parent", homeMode: "isolated" }, + { outcome: "created", scenario: "interrupted_bootstrap_unopened", homeMode: "isolated" }, + { outcome: "created", scenario: "interrupted_bootstrap_unopened_active_turn", homeMode: "isolated" }, + { outcome: "created", scenario: "interrupted_bootstrap_unopened_unproven", homeMode: "isolated" }, ])( "bootstraps only an untouched provider session in a $outcome workspace lease: $scenario ($homeMode)", async ({ outcome, scenario, homeMode }) => { const remoteCwd = join(isolatedStateDirectory, "remote"); @@ -20321,11 +20327,19 @@ describe("runnerd provider runtime wiring", () => { try { execFileSync("sh", command.args, { stdio: "pipe" }); return { exitCode: 0, timedOut: false, stdout: "", stderr: "" }; } catch { return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; } } + if (command.args?.[1]?.includes("base64") && scenario.startsWith("interrupted_bootstrap_unopened") + && command.args[1].includes("codex-provider-state.json") && !command.args[1].includes("runner-state.json")) return { + exitCode: 0, timedOut: false, stderr: "", stdout: Buffer.from(JSON.stringify({ + schema: "paperclip.runner.codex-provider-state.v1", lifecycle: "prepared", threadId: null, providerSessionId: null, + activeProviderTurnId: scenario.endsWith("_active_turn") ? "turn-in-progress" : null, ambiguousTurnStartPending: false, + config: { provider: "codex", driver: "codex_app_server" } })).toString("base64"), + }; if (command.args?.[1]?.includes("base64") && scenario.startsWith("interrupted_bootstrap")) return { exitCode: 0, timedOut: false, stderr: "", stdout: Buffer.from(JSON.stringify({ runnerInstanceId: "runner-new-in-retained-workspace", normalizedSessionId: execution.session.normalizedSessionId, - runId: scenario === "interrupted_bootstrap_other_run" ? "previous-warm-run" : execution.binding.runId, - lifecycle: "ready", outbox: [], pendingTerminalDelivery: null })).toString("base64"), + runId: scenario.endsWith("_other_run") ? "previous-warm-run" : execution.binding.runId, + lifecycle: scenario.startsWith("interrupted_bootstrap_unopened") ? "suspended" : "ready", + outbox: [], pendingTerminalDelivery: null })).toString("base64"), }; if (command.args?.[1]?.includes("base64")) return { exitCode: 1, timedOut: false, stdout: "", stderr: "", @@ -20416,10 +20430,11 @@ describe("runnerd provider runtime wiring", () => { await mkdir(join(options.stateDirectory!, "failover-backups", "current"), { recursive: true }); await writeFile(join(options.stateDirectory!, "failover-backups", "current", "manifest.json"), "{}"); } - if (["interrupted_bootstrap", "interrupted_bootstrap_other_run", "interrupted_bootstrap_symlink_parent"].includes(scenario)) { + if (["interrupted_bootstrap", "interrupted_bootstrap_other_run", "interrupted_bootstrap_symlink_parent", + "interrupted_bootstrap_unopened", "interrupted_bootstrap_unopened_other_run", "interrupted_bootstrap_unopened_active_turn"].includes(scenario)) { noteRemoteBootstrapRestartProven(execution.binding.runId, "stopped"); } - if (scenario === "interrupted_bootstrap") { + if (scenario === "interrupted_bootstrap" || scenario === "interrupted_bootstrap_unopened") { await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); const siblings = await readdir(dirname(sessionRoot)); expect(siblings.filter(name => name.startsWith(`${basename(sessionRoot)}.bootstrap-restart-`))).toHaveLength(1); @@ -21248,6 +21263,30 @@ describe("HTTP Stop during remote runner staging", () => { }, 120000); }); +describe("remoteProviderStateNeverOpened", () => { + const acpx = { provider: { kind: "acpx" } } as unknown as Parameters[0]; + const prepared = { + schema: "paperclip.runner.acpx-provider-state.v3", launchProfileDigest: `sha256:${"c".repeat(64)}`, lifecycle: "prepared", + descriptor: { kind: "acpx", provider: "acpx", driver: "acpx_runtime", agent: "claude", model: "explicit-model" }, + toolSet: { schema: "paperclip.runner.authorized-tools.v1", schemaVersion: 1, catalogDigest: `sha256:${"d".repeat(64)}`, operations: [] }, + identity: null, activeTurnId: null, attemptedTurnControls: [], providerExitUnconfirmed: false, + semanticResult: null, goalProjection: null, turnControls: { steering: false, queuedFollowUp: false }, + goalRevision: 0, goalSourceRevision: null, pendingEvents: [], pendingRuntimeRequests: {}, nextEventSequence: 1, + openSemanticCalls: [], retiredSemanticCalls: [], + }; + it("accepts the ACPX state a rejected session.open leaves, and nothing that may hold a turn", () => { + expect(remoteProviderStateNeverOpened(acpx, prepared)).toBe(true); + const { providerExitUnconfirmed: _exit, activeTurnId: _turn, identity: _identity, ...beforeDefaults } = prepared; + expect(remoteProviderStateNeverOpened(acpx, beforeDefaults)).toBe(true); + for (const changed of [{ lifecycle: "session_open" }, { lifecycle: "turn_active", activeTurnId: "turn" }, { activeTurnId: "turn" }, + { providerExitUnconfirmed: true }, { lifecycle: "closed", providerExitUnconfirmed: true }, { schema: "paperclip.runner.acpx-provider-state.v2" }, + { identity: { kind: "acpx", acpxRecordId: "record" } }]) { + expect(remoteProviderStateNeverOpened(acpx, { ...prepared, ...changed })).toBe(false); + } + expect(remoteProviderStateNeverOpened({ provider: { kind: "claude_managed" } } as never, { lifecycle: "prepared" })).toBe(false); + }); +}); + describe("isTransientRemoteCheckpointReadFailure", () => { it("repeats only a dropped sandbox connection, never a state or safety refusal", () => { expect(isTransientRemoteCheckpointReadFailure(Object.assign(new Error("write ECONNRESET"), { code: "ECONNRESET" }))).toBe(true); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 7ed7a9692c6..ac9a832446a 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -7607,6 +7607,24 @@ function providerSessionIdentityIsPresent(value: unknown): boolean { ); } +export function remoteProviderStateNeverOpened(execution: NativeExecutionInput, providerState: unknown): boolean { + const state = record(providerState); + if (state.lifecycle !== "prepared") return false; + switch (execution.provider.kind) { + case "acpx": + return state.schema === ACPX_PROVIDER_STATE_SCHEMA && (state.identity ?? null) === null + && (state.activeTurnId ?? null) === null && (state.providerExitUnconfirmed ?? false) === false; + case "cursor_cloud": + case "codex": + case "opencode": + return state.schema === CODEX_PROVIDER_STATE_SCHEMA && (state.threadId ?? null) === null + && (state.providerSessionId ?? null) === null && (state.activeProviderTurnId ?? null) === null + && state.ambiguousTurnStartPending !== true; + default: + return false; + } +} + /** Closed diagnostic categories; durable state may contain private provider data. */ export function durableProviderCheckpointFailureReason(execution: NativeExecutionInput, providerState: unknown): string { if (execution.provider.kind !== "acpx") return "identity_binding"; @@ -14671,7 +14689,7 @@ export { resolveRemoteRunnerTransportMode } from "./native-runner-execution-driv export function remoteCheckpointIncompleteFailure( settlement: "settled" | "unsettled", - incompleteReason: "unavailable" | "not_suspended" | null, + incompleteReason: "unavailable" | "not_suspended" | "provider_unopened" | null, ): Error | null { // A transport that never completed provider bootstrap has no provider state // to preserve; its original launch failure remains authoritative. Once @@ -15740,13 +15758,20 @@ async function createRunnerdBackendWithinSessionClaim( * this invocation's archived rotation snapshot; every other caller, * including close checkpointing, must inspect the live Runner state. */ + const provenOwnBootstrapRestart = (runId: string | null | undefined) => + input.restartRecovery?.kind === "bootstrap_incomplete" + && runId === input.execution.binding.runId + && hasRemoteBootstrapRestartProof(input.execution.binding.runId); + const restartableBootstrapState = (state: { incompleteReason: string | null; notSuspendedRunId?: string | null }) => + (state.incompleteReason === "not_suspended" || state.incompleteReason === "provider_unopened") + && provenOwnBootstrapRestart(state.notSuspendedRunId); const inspectRemoteHarnessState = async ( source: "live" | "host_preparation" = "live", ): Promise<{ complete: boolean; runnerState: Record | null; providerSessionIdentity: Record | null; - incompleteReason: "unavailable" | "not_suspended" | null; + incompleteReason: "unavailable" | "not_suspended" | "provider_unopened" | null; notSuspendedRunId?: string | null; }> => { if (!remoteCommandRunner || !remoteStateDirectory) { @@ -15834,6 +15859,16 @@ async function createRunnerdBackendWithinSessionClaim( providerState, }); if (!providerSessionIdentityIsPresent(providerSessionIdentity)) { + if (source === "host_preparation" && remoteProviderStateNeverOpened(input.execution, providerState) + && provenOwnBootstrapRestart(typeof runnerState.runId === "string" ? runnerState.runId : null)) { + return { + complete: false, + runnerState: null, + providerSessionIdentity: null, + incompleteReason: "provider_unopened", + notSuspendedRunId: input.execution.binding.runId, + }; + } throw new Error(`runner_harness_state_mismatch: provider_identity_incomplete (${durableProviderCheckpointFailureReason(input.execution, providerState)})`); } const previousManifest = compatibleNativeHarnessBackupManifests({ @@ -16173,10 +16208,7 @@ async function createRunnerdBackendWithinSessionClaim( !state.runnerState || !state.providerSessionIdentity ) { - const provenRestart = state.incompleteReason === "not_suspended" && !backupAvailable - && input.restartRecovery?.kind === "bootstrap_incomplete" - && state.notSuspendedRunId === input.execution.binding.runId - && hasRemoteBootstrapRestartProof(input.execution.binding.runId) + const provenRestart = restartableBootstrapState(state) && !backupAvailable && await moveInterruptedBootstrapSessionAside() && takeRemoteBootstrapRestartProof(input.execution.binding.runId); // A resumed allocation that never held this session (for example @@ -16255,10 +16287,7 @@ async function createRunnerdBackendWithinSessionClaim( // lease was not provider-confirmed lost. Never silently create a new // provider session from that ambiguous state. throw new Error("runner_harness_state_mismatch: backup_without_reusable_lease"); - } else if (state.incompleteReason === "not_suspended" - && input.restartRecovery?.kind === "bootstrap_incomplete" - && state.notSuspendedRunId === input.execution.binding.runId - && hasRemoteBootstrapRestartProof(input.execution.binding.runId)) { + } else if (restartableBootstrapState(state)) { if (!(await moveInterruptedBootstrapSessionAside() && takeRemoteBootstrapRestartProof(input.execution.binding.runId) && await claimUntouchedRemoteSession())) {