From d50268c932da9e03c304f6a48cdaa5ee477331f2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 10 Oct 2026 19:12:54 -0500 Subject: [PATCH] fix: classify computer checkpoint recovery rejection safely Preserve every settlement and ownership gate while exposing fixed reason codes for checkpoint mismatches and prior authority verification. Cover each diagnostic branch without logging evidence values. Co-Authored-By: Paperclip --- .../native-session-executor.test.ts | 53 ++++++++- .../native-runtime/native-session-executor.ts | 105 ++++++++++++------ 2 files changed, 126 insertions(+), 32 deletions(-) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 1539c1204e..895e6e4fee 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1,4 +1,4 @@ -import { compareAndSealRetiredComputerCheckpoint, retiredComputerCheckpointIsSettled, retireUnlaunchedComputerAttempt, nativeSessionBootstrapTimeoutMs } from "./native-session-executor.js"; +import { compareAndSealRetiredComputerCheckpoint, retiredComputerCheckpointIsSettled, retiredComputerCheckpointUnsettledReason, retireUnlaunchedComputerAttempt, nativeSessionBootstrapTimeoutMs } from "./native-session-executor.js"; import { prepareHeartbeatGitHubLaunchers } from "../heartbeat-github-launchers.js"; import { gunzipSync } from "node:zlib"; import { configuredEnvironmentProjection } from "../../vendor/paperclip-runner/index.js"; @@ -13970,6 +13970,57 @@ describe("computer idle authority transition", () => { expect(retiredComputerCheckpointIsSettled(input)).toBe(false); }); + it("reports no diagnostic for settled evidence", () => { + expect(retiredComputerCheckpointUnsettledReason(settled())).toBeNull(); + }); + + type SettlementInput = Parameters[0]; + it.each<{ reason: string; change: (input: SettlementInput) => void }>([ + { reason: "provider_kind", change: input => { input.execution = { ...input.execution, provider: { ...input.execution.provider, kind: "acpx" } } as typeof input.execution; } }, + { reason: "runner_schema", change: input => { input.runner.schema = "unknown"; } }, + { reason: "runner_lifecycle", change: input => { input.runner.lifecycle = "running"; } }, + { reason: "runner_identity", change: input => { input.runner.runId = "private-mismatched-value"; } }, + { reason: "controller_identity", change: input => { input.control.identity = { ...input.identity, runnerInstanceId: "private-mismatched-value" }; } }, + { reason: "runner_outbox", change: input => { input.runner.outbox = [{ secret: "private-unacknowledged-output" }]; } }, + { reason: "runner_terminal_delivery", change: input => { input.runner.pendingTerminalDelivery = { secret: "private-terminal-output" }; } }, + { reason: "controller_commands", change: input => { input.control.commands = [{ status: "pending", input: "private-command" }]; } }, + { reason: "controller_terminal_event", change: input => { input.control.committedEvents = [{ eventType: "run.started" }]; } }, + { reason: "controller_terminal_identity", change: input => { input.control.committedEvents = [{ eventType: "run.terminal", envelope: { ...input.identity, runId: "private-other-run" } }]; } }, + { reason: "provider_terminal_authority", change: input => { input.provider.completedTurnAuthoritative = false; } }, + { reason: "prior_run_identity", change: input => { input.priorExecution = { ...input.priorExecution, binding: { ...input.priorExecution.binding, runId: "private-other-run" } }; } }, + { reason: "issue_scope", change: input => { input.execution = { ...input.execution, binding: { ...input.execution.binding, issueId: "private-other-issue" } }; } }, + { reason: "session_scope", change: input => { input.execution = { ...input.execution, session: { ...input.execution.session, normalizedSessionId: "private-other-session" } } as typeof input.execution; } }, + { reason: "session_config", change: input => { input.execution = { ...input.execution, provider: { ...input.execution.provider, model: "private-changed-model" } } as typeof input.execution; } }, + { reason: "provider_session_identity", change: input => { input.provider.threadId = null; } }, + ])("classifies $reason without exposing evidence values or accepting recovery", ({ reason, change }) => { + const input: SettlementInput = settled(); + change(input); + expect(retiredComputerCheckpointUnsettledReason(input)).toBe(reason); + expect(retiredComputerCheckpointIsSettled(input)).toBe(false); + }); + + describe.each(["prior", "current"] as const)("%s checkpoint diagnostics", source => { + it.each([ + ["provider_session", "providerSessionId"], + ["driver", "driverKind"], + ["active_turn", "activeTurnId"], + ["runtime_requests", "pendingRuntimeRequests"], + ["run_identity", "runId"], + ["session_identity", "sessionId"], + ["company_identity", "companyId"], + ["agent_identity", "agentId"], + ["issue_identity", "issueId"], + ] as const)("identifies %s separately from the other checkpoint", (reason, field) => { + const input: SettlementInput = settled(); + const key = source === "prior" ? "checkpoint" : "currentCheckpoint"; + input[key] = reason.endsWith("_identity") + ? { ...input[key], identity: { ...input[key].identity as Record, [field]: "private-mismatched-value" } } + : { ...input[key], [field]: "private-mismatched-value" }; + expect(retiredComputerCheckpointUnsettledReason(input)).toBe(`${source}_checkpoint_${reason}`); + expect(retiredComputerCheckpointIsSettled(input)).toBe(false); + }); + }); + it.each(["unchanged", "runner-changed", "provider-changed", "symlink", "interrupted"])( "atomically seals only the unchanged remote bytes (%s)", async (condition) => { const root = await mkdtemp(join(tmpdir(), "computer-retired-cas-")); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 0b54687fd1..9052fd31d0 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -10891,10 +10891,7 @@ export async function syncRemoteRunnerDirectoryOut(input: { } } -/** A dead computer runner may retain a ready checkpoint after controller loss. - * Match the local quiescent-recovery proof before sealing it for normal rotation. - */ -export function retiredComputerCheckpointIsSettled(input: { +type RetiredComputerCheckpointInput = { execution: NativeExecutionInput; priorExecution: NativeExecutionInput; identity: RunnerdDurableIdentity; @@ -10903,34 +10900,77 @@ export function retiredComputerCheckpointIsSettled(input: { provider: Record; checkpoint: Record; currentCheckpoint: Record; -}): boolean { +}; + +type RetiredComputerCheckpointUnsettledReason = + | "provider_kind" + | "runner_schema" + | "runner_lifecycle" + | "runner_identity" + | "controller_identity" + | "runner_outbox" + | "runner_terminal_delivery" + | "controller_commands" + | "controller_terminal_event" + | "controller_terminal_identity" + | "provider_terminal_authority" + | "prior_run_identity" + | "issue_scope" + | "session_scope" + | "session_config" + | "provider_session_identity" + | `${"prior" | "current"}_checkpoint_${"provider_session" | "driver" | "active_turn" | "runtime_requests" | "run_identity" | "session_identity" | "company_identity" | "agent_identity" | "issue_identity"}`; + +/** Fixed reason codes are safe to surface in run logs; never include checkpoint + * values, provider configuration, prompts, paths, or credentials in diagnostics. */ +export function retiredComputerCheckpointUnsettledReason( + input: RetiredComputerCheckpointInput, +): RetiredComputerCheckpointUnsettledReason | null { const { execution, priorExecution, identity, runner, control, provider, checkpoint, currentCheckpoint } = input; - if (execution.provider.kind !== "codex" || priorExecution.provider.kind !== "codex") return false; + if (execution.provider.kind !== "codex" || priorExecution.provider.kind !== "codex") return "provider_kind"; const events = control.committedEvents; const commands = control.commands; const terminal = record(record(Array.isArray(events) ? events.at(-1) : null).envelope); const fields = ["runId", "normalizedSessionId", "runnerInstanceId", "environmentLeaseId", "turnId", "itemId"] as const; - if (runner.schema !== RUNNERD_STATE_SCHEMA || runner.lifecycle !== "ready" || - fields.some(key => runner[key] !== identity[key] || record(control.identity)[key] !== identity[key]) || - !Array.isArray(runner.outbox) || runner.outbox.length !== 0 || runner.pendingTerminalDelivery !== null || - !Array.isArray(commands) || commands.some(command => record(command).status !== "completed") || - !Array.isArray(events) || record(events.at(-1)).eventType !== "run.terminal" || - terminal.runId !== identity.runId || terminal.normalizedSessionId !== identity.normalizedSessionId || - provider.completedTurnAuthoritative !== true || priorExecution.binding.runId !== identity.runId || - priorExecution.binding.issueId !== execution.binding.issueId || - nativeSessionScopeKey(priorExecution) !== nativeSessionScopeKey(execution) || - nativeSessionConfigDigest(priorExecution) !== nativeSessionConfigDigest(execution)) return false; + if (runner.schema !== RUNNERD_STATE_SCHEMA) return "runner_schema"; + if (runner.lifecycle !== "ready") return "runner_lifecycle"; + if (fields.some(key => runner[key] !== identity[key])) return "runner_identity"; + if (fields.some(key => record(control.identity)[key] !== identity[key])) return "controller_identity"; + if (!Array.isArray(runner.outbox) || runner.outbox.length !== 0) return "runner_outbox"; + if (runner.pendingTerminalDelivery !== null) return "runner_terminal_delivery"; + if (!Array.isArray(commands) || commands.some(command => record(command).status !== "completed")) return "controller_commands"; + if (!Array.isArray(events) || record(events.at(-1)).eventType !== "run.terminal") return "controller_terminal_event"; + if (terminal.runId !== identity.runId || terminal.normalizedSessionId !== identity.normalizedSessionId) return "controller_terminal_identity"; + if (provider.completedTurnAuthoritative !== true) return "provider_terminal_authority"; + if (priorExecution.binding.runId !== identity.runId) return "prior_run_identity"; + if (priorExecution.binding.issueId !== execution.binding.issueId) return "issue_scope"; + if (nativeSessionScopeKey(priorExecution) !== nativeSessionScopeKey(execution)) return "session_scope"; + if (nativeSessionConfigDigest(priorExecution) !== nativeSessionConfigDigest(execution)) return "session_config"; const providerIdentity = providerSessionIdentityFromDurableProviderState({ execution: priorExecution, providerState: provider }); - if (!providerIdentity.providerSessionId) return false; - return [checkpoint, currentCheckpoint].every((value, index) => { + if (!providerIdentity.providerSessionId) return "provider_session_identity"; + for (const [source, value, expectedRunId] of [ + ["prior", checkpoint, identity.runId], + ["current", currentCheckpoint, execution.binding.runId], + ] as const) { const binding = record(value.identity); - return value.providerSessionId === providerIdentity.providerSessionId && - value.driverKind === priorExecution.session.driverKind && value.activeTurnId === null && - Array.isArray(value.pendingRuntimeRequests) && value.pendingRuntimeRequests.length === 0 && - binding.runId === (index === 0 ? identity.runId : execution.binding.runId) && - binding.sessionId === identity.normalizedSessionId && binding.companyId === execution.binding.companyId && - binding.agentId === execution.binding.agentId && binding.issueId === execution.binding.issueId; - }); + if (value.providerSessionId !== providerIdentity.providerSessionId) return `${source}_checkpoint_provider_session`; + if (value.driverKind !== priorExecution.session.driverKind) return `${source}_checkpoint_driver`; + if (value.activeTurnId !== null) return `${source}_checkpoint_active_turn`; + if (!Array.isArray(value.pendingRuntimeRequests) || value.pendingRuntimeRequests.length !== 0) return `${source}_checkpoint_runtime_requests`; + if (binding.runId !== expectedRunId) return `${source}_checkpoint_run_identity`; + if (binding.sessionId !== identity.normalizedSessionId) return `${source}_checkpoint_session_identity`; + if (binding.companyId !== execution.binding.companyId) return `${source}_checkpoint_company_identity`; + if (binding.agentId !== execution.binding.agentId) return `${source}_checkpoint_agent_identity`; + if (binding.issueId !== execution.binding.issueId) return `${source}_checkpoint_issue_identity`; + } + return null; +} + +/** A dead computer runner may retain a ready checkpoint after controller loss. + * Match the local quiescent-recovery proof before sealing it for normal rotation. + */ +export function retiredComputerCheckpointIsSettled(input: RetiredComputerCheckpointInput): boolean { + return retiredComputerCheckpointUnsettledReason(input) === null; } async function sealRetiredComputerRunnerState(input: { @@ -10969,12 +11009,15 @@ print(json.dumps(rows))`, runnerPath, providerPath], bypassSession: true, timeou throw new Error("runner_remote_retirement_authority_unverified"); const profile = record(prior.runnerProfileJson); const previous = parseNativeExecutionInput(profile.nativeExecutionInput); - if (!retiredComputerCheckpointIsSettled({ execution: input.execution, priorExecution: previous, identity, runner, - control: record(JSON.parse(controlBytes.toString("utf8"))), provider: record(JSON.parse(providerBytes!.toString("utf8"))), - checkpoint: record(profile.sessionCheckpoint), currentCheckpoint: record(record(current.runnerProfileJson).sessionCheckpoint) }) || - await verifyPriorRunnerdStateForSessionScope({ db: input.db, root: input.root, identity, execution: input.execution, - allowVerifiedBackup: false, allowRetainedWarmRunner: false, remoteRunnerState: true, runnerExecutionTarget: input.target }) !== "verified") - throw new Error("runner_remote_retirement_checkpoint_unsettled"); + const unsettledReason = retiredComputerCheckpointUnsettledReason({ execution: input.execution, priorExecution: previous, identity, runner, + control: record(JSON.parse(controlBytes.toString("utf8"))), provider: record(JSON.parse(providerBytes!.toString("utf8"))), + checkpoint: record(profile.sessionCheckpoint), currentCheckpoint: record(record(current.runnerProfileJson).sessionCheckpoint) }); + if (unsettledReason !== null) + throw new Error(`runner_remote_retirement_checkpoint_unsettled: ${unsettledReason}`); + const priorAuthority = await verifyPriorRunnerdStateForSessionScope({ db: input.db, root: input.root, identity, execution: input.execution, + allowVerifiedBackup: false, allowRetainedWarmRunner: false, remoteRunnerState: true, runnerExecutionTarget: input.target }); + if (priorAuthority !== "verified") + throw new Error(`runner_remote_retirement_checkpoint_unsettled: prior_authority_${priorAuthority}`); // Recheck exact durable retirement after all awaited reads; never infer death // from a missing PID or turn status. The compare-and-swap preserves ambiguity. const workspace = readNativeComputerWorkspaceReference(profile.nativeComputerWorkspace);