diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 29e7669f97e..9ac6842d450 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -347,6 +347,14 @@ For a prepared remote workspace interrupted before its first executor claim, that ledger records `remote_execution_never_claimed` with a `bootstrap_incomplete` disposition. The provider attempt remains zero during classification; the executor increments it before starting provider work. +A claimed remote bootstrap (the sandbox runner was launched and may have +connected) records `remote_bootstrap_proven_incomplete` with the same +disposition. Recovery admits it only after the previous controller lease has +expired and the Postgres controller checkpoint holds just `run.prepare` and an +unfinished `session.open`, with no committed or acknowledged event. The +executor restores that checkpoint and checks the same proof before it +bootstraps again. After three provider attempts it records +`execution_recovery_budget_exhausted` instead. For a local Codex, OpenCode, or ACPX turn lost while runnerd was stopped, the canonical `turn.failed` event retains `error.code: "provider_turn_lost_on_restore"` and diff --git a/server/src/services/native-runtime/native-bootstrap-proof.ts b/server/src/services/native-runtime/native-bootstrap-proof.ts new file mode 100644 index 00000000000..786db4d3034 --- /dev/null +++ b/server/src/services/native-runtime/native-bootstrap-proof.ts @@ -0,0 +1,68 @@ +/** + * Pure proofs over durable runner state that a bootstrap never reached + * provider work. Restart recovery and the executor share them, so the claim + * and the executor's own check accept exactly the same shapes. + */ + +export const RUNNERD_CONTROL_PLANE_STATE_SCHEMAS: ReadonlySet = new Set([ + "paperclip.runner.durable.control-plane-state.v1", + "paperclip.runner.durable.control-plane-state.v2", +]); + +const RUNNERD_BOOTSTRAP_COMMAND_TYPES = new Set(["run.prepare", "session.open"]); + +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +/** The runner never connected, so every bootstrap command is still undelivered. */ +export function runnerdStateIsUnconnectedBootstrap(state: Record): boolean { + const commands = Array.isArray(state.commands) ? state.commands.map(record) : []; + const committedEvents = Array.isArray(state.committedEvents) ? state.committedEvents : []; + return RUNNERD_CONTROL_PLANE_STATE_SCHEMAS.has(String(state.schema)) && + state.connectionCount === 0 && committedEvents.length === 0 && + commands.every(command => command.status === "pending" && + RUNNERD_BOOTSTRAP_COMMAND_TYPES.has(String(command.type))); +} + +/** + * A provider turn can start only after session.open completes and the + * controller issues a turn command. A root whose controller journal stops + * before that point (run.prepare may be acknowledged, session.open never is, + * no event was committed or acknowledged) and whose runner journal emitted no + * event either holds no provider work, so restart recovery may quarantine it + * and bootstrap again. Any other shape stays fail-closed. + * + * `runnerJournal` is `undefined` when the runner journal is not on this host + * (never written locally, or a remote runner), `null` when it is unreadable. + */ +export function runnerdStateIsIncompleteBootstrap( + state: Record, + runnerJournal?: Record | null, +): boolean { + if (runnerdStateIsUnconnectedBootstrap(state)) return true; + if (!RUNNERD_CONTROL_PLANE_STATE_SCHEMAS.has(String(state.schema))) return false; + if (typeof state.connectionCount !== "number" || !Number.isSafeInteger(state.connectionCount) || + state.connectionCount < 0) return false; + if (!Array.isArray(state.committedEvents) || state.committedEvents.length !== 0) return false; + if (state.ackedSourceSeq !== 0) return false; + if (state.warmTransition != null || state.completedWarmTransition != null || + state.stagedRunAttachment != null) return false; + if (record(state.bootstrapHistory).adopted === true) return false; + if (!Array.isArray(state.commands)) return false; + const commands = state.commands.map(record); + if (!commands.every(command => RUNNERD_BOOTSTRAP_COMMAND_TYPES.has(String(command.type)) && + (command.status === "pending" || + (command.type === "run.prepare" && command.status === "completed")))) return false; + // The controller cannot see runner events that were emitted but never + // committed. When the runner journal exists it must agree that none were. + if (runnerJournal === undefined) return true; + if (runnerJournal === null) return false; + const processed = Object.values(record(runnerJournal.processedCommands)).map(record); + return Array.isArray(runnerJournal.outbox) && runnerJournal.outbox.length === 0 && + runnerJournal.nextSourceSeq === 1 && runnerJournal.ackedSourceSeq === 0 && + processed.every(command => RUNNERD_BOOTSTRAP_COMMAND_TYPES.has(String(command.commandType)) && + (command.commandType === "run.prepare" || command.status !== "completed")); +} diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 1baceb4e514..d750a83a141 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -11,7 +11,8 @@ import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-pre import { notifyInstructionCollection } from "../instruction-collection-notifications.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { preserveNativeFailedWorkspace } from "./native-failed-workspace-recovery.js"; -import { hasNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader } from "./native-runner-checkpoint.js"; +import { hasNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader, + nativeRunnerCheckpointProvesIncompleteBootstrap } from "./native-runner-checkpoint.js"; import { resolveNativeStagedAttachmentRecovery, type NativeStagedAttachmentRecovery } from "./native-staged-attachment-recovery.js"; import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js"; import { activeNativeExecutionRunIds, NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js"; @@ -340,6 +341,8 @@ export function classifyNativeRunnerRecoveryEvidence(input: { runnerGroupAlive: boolean; remoteSandbox?: boolean; unclaimedBootstrap?: boolean; + /** The durable controller checkpoint of a claimed remote bootstrap proves no provider work. */ + remoteBootstrapProvenIncomplete?: boolean; processStartMatches: boolean; knownProviderProcessAlive?: boolean; knownProviderProcessIdentityAmbiguous?: boolean; @@ -368,6 +371,12 @@ export function classifyNativeRunnerRecoveryEvidence(input: { if (input.unclaimedBootstrap === true && !input.hasCheckpoint && !input.hasProviderEvidence) { return { claimKind: "bootstrap_incomplete", reason: "remote_execution_never_claimed" }; } + if (input.remoteBootstrapProvenIncomplete === true && !input.hasCheckpoint && !input.hasProviderEvidence) { + // Each same-run bootstrap is another provider attempt; keep restarts bounded. + if ((input.providerAttempt ?? 0) >= 3) + return { claimKind: null, reason: "execution_recovery_budget_exhausted" }; + return { claimKind: "bootstrap_incomplete", reason: "remote_bootstrap_proven_incomplete" }; + } return input.hasCheckpoint && input.checkpointIdentityMatches === true && input.hasProviderEvidence @@ -1017,9 +1026,23 @@ export async function claimNativeRestartRecoveries(input: { unclaimedBootstrap = !executionEvent; } + // A claimed remote bootstrap (runner launched, perhaps connected) has no + // provider evidence when its controller dies before session.open + // completes. The owner's lease has expired above, and the controller + // writes every command to Postgres before sending it, so that checkpoint + // is the complete record of what reached the sandbox runner. When it + // proves no provider work, bootstrap again instead of holding the task. + const remoteBootstrapProvenIncomplete = !unclaimedBootstrap && remoteWorkspace !== null && !pluginTask + && Boolean(durableAuthority) && stagedAttachment === null && !hasCheckpoint && !hasProviderEvidence + && Boolean(row.run.runnerInstanceId && row.run.nativeSessionId) + && await nativeRunnerCheckpointProvesIncompleteBootstrap(tx, { + companyId: row.run.companyId, issueId: row.coordinator.issueId, runId: row.run.id, + }, { runnerInstanceId: row.run.runnerInstanceId!, normalizedSessionId: row.run.nativeSessionId! }); + const classification = classifyNativeRunnerRecoveryEvidence({ remoteSandbox: remoteSandbox || Boolean(pluginTask && durableAuthority), unclaimedBootstrap, + remoteBootstrapProvenIncomplete, runnerPidAlive, runnerGroupAlive, processStartMatches: exactRunnerIdentity, diff --git a/server/src/services/native-runtime/native-runner-checkpoint.ts b/server/src/services/native-runtime/native-runner-checkpoint.ts index 9e7340553d5..f72e9123c6c 100644 --- a/server/src/services/native-runtime/native-runner-checkpoint.ts +++ b/server/src/services/native-runtime/native-runner-checkpoint.ts @@ -5,6 +5,7 @@ import { inspectDurablePrpCheckpointHeader, inspectUnclaimedBootstrapCheckpoint, import { createLocalEncryptedCipher } from "../../secrets/local-encrypted-provider.js"; import { legacyControllerBootId } from "../legacy-controller-lease.js"; import { assertNativeSourceCheckpointConsumer, lockSourceRecoveryCoordinators } from "./native-source-recovery-ownership.js"; +import { runnerdStateIsIncompleteBootstrap } from "./native-bootstrap-proof.js"; type Binding = { companyId: string; issueId: string; runId: string }; export const NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES = 1024 * 1024; @@ -248,6 +249,53 @@ export async function loadNativeUnclaimedBootstrapCheckpoint( } catch { return null; } } +const BOOTSTRAP_CHECKPOINT_PARTS = new Set(["header", "cursor", "command:1", "command:2"]); + +/** + * A remote runner's controller state lives only in this checkpoint, and every + * command is written here before the controller sends it. Once the previous + * controller's lease has expired, the checkpoint is therefore a complete record + * of what reached the sandbox runner. Returns true only when that record holds + * the bootstrap commands alone (run.prepare, an unfinished session.open) with + * no committed or acknowledged event, so no provider turn can exist. The + * sandbox runner journal is not on this host; the proof uses the controller + * record alone, as for a local root whose runner never journaled. + */ +export async function nativeRunnerCheckpointProvesIncompleteBootstrap( + db: Pick, binding: Binding, expected: { runnerInstanceId: string; normalizedSessionId: string }, +): Promise { + try { + const parts = await db.select({ part: nativeRunnerCheckpoints.part }).from(nativeRunnerCheckpoints).where(and( + eq(nativeRunnerCheckpoints.companyId, binding.companyId), eq(nativeRunnerCheckpoints.issueId, binding.issueId), + eq(nativeRunnerCheckpoints.runId, binding.runId))).limit(BOOTSTRAP_CHECKPOINT_PARTS.size + 1); + if (!parts.length || parts.length > BOOTSTRAP_CHECKPOINT_PARTS.size + || parts.some(row => !BOOTSTRAP_CHECKPOINT_PARTS.has(row.part))) return false; + const checkpoint = await loadNativeRunnerCheckpoint(db, binding); + if (!checkpoint || checkpoint.predecessor !== null + || checkpoint.recoveryEvidence.runnerInstanceId !== expected.runnerInstanceId + || checkpoint.recoveryEvidence.normalizedSessionId !== expected.normalizedSessionId) return false; + // The decoder above validated these canonical v2 records. Rebuild the + // controller state in memory; the part check admits no event records. + const read = (part: string) => { + const value = checkpoint.records.get(part); + return value === undefined ? undefined : JSON.parse(value) as Record; + }; + const header = read("header"); + const cursor = read("cursor"); + if (header?.schema !== "paperclip.runner.controller-checkpoint.v2" || !cursor) return false; + const state: Record = { + ...(header.state as Record), + ackedSourceSeq: cursor.ackedSourceSeq, + commands: ["command:1", "command:2"].map(read).filter(command => command !== undefined), + committedEvents: [], + }; + return (state.identity as { runId?: unknown } | undefined)?.runId === binding.runId + && runnerdStateIsIncompleteBootstrap(state); + } catch { + return false; + } +} + export async function restoreNativeRunnerCheckpoint(db: Db, binding: Binding, root: string, normalizedSessionId?: string) { const checkpoint = await loadNativeRunnerCheckpoint(db, binding); if (!checkpoint) return null; diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 5241fe2b5b5..a9cc8f325cc 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -60,7 +60,10 @@ import { detachNativeSessionsForRestart, executePaperclipNativeSession, NativeControllerDetachedForRestartError, + restoreRemoteBootstrapCheckpointForRecovery, + runnerdStateProvesIncompleteBootstrap, } from "./native-session-executor.js"; +import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { listExpiredNativeRunnerAuthorities, loadNativeRunnerCheckpoint, @@ -2220,6 +2223,140 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( }, ); + // RC-29: a restart while a Daytona runner was connecting. The executor had + // claimed the run and launched the sandbox runner, which authenticated, but + // session.open never completed. The controller's Postgres checkpoint is the + // only record of what the runner received. + async function seedClaimedRemoteBootstrap(label: string, options: { + attempt?: number; + mutate?: (core: DurablePrpControlPlane) => void; + checkpoint?: boolean; + } = {}) { + const fixture = await seedRun(`REMOTE-CLAIMED-${label}`); + const { runId, issueId, native } = fixture; + const remote = remotePreparedWorkspace(); + await fixture.db.update(heartbeatRuns).set({ + runnerProfileJson: { ...fixture.preparedProfile, nativeWorkspaceSync: remote }, + processPid: 443, processStartedAt: new Date(Date.now() - 30_000), + }).where(eq(heartbeatRuns.id, runId)); + await fixture.db.insert(heartbeatRunEvents).values([ + { companyId, runId, agentId, seq: 900, stream: "system", level: "info", eventType: "native.process_start_requested" }, + { companyId, runId, agentId, seq: 901, stream: "system", level: "info", eventType: "native.process_identity_recorded", + payload: { targetKind: "remote", processPid: 443 } }, + ]); + const bootId = randomUUID(); + const leaseOwner = `${bootId}:1:${randomUUID()}`; + await fixture.db.update(nativeRunFinalizations).set({ + attempt: options.attempt ?? 1, controllerGeneration: 1, controllerBootId: bootId, controllerPid: 2_000_000_001, + controllerProcessStartedAt: new Date(Date.now() - 60_000), leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000), + }).where(eq(nativeRunFinalizations.runId, runId)); + if (options.checkpoint !== false) { + const identity = { runnerInstanceId: native.runnerInstanceId, environmentLeaseId: runId, runId, + normalizedSessionId: native.normalizedSessionId, turnId: `turn-${runId}`, itemId: `item-${runId}` }; + const core = new DurablePrpControlPlane({ stateDirectory: resolve(runtimeRoot, runId, "control-plane"), identity, + expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + try { + const template = { provider: { provider: "acpx", command: "fixture" }, authorizedTools: {} }; + core.persistRunAttachTemplate(template); + core.queueCommand("run.prepare", template); + core.queueCommand("session.open", { reuse: "same_session" }); + core.issueBootstrapTicket(); + // The runner authenticated once and received run.prepare. + core.store.state.connectionCount = 1; + core.store.state.freshBootstraps = 1; + core.store.state.commandDeliveryCounts = { [core.store.state.commands[0]!.commandId]: 1 }; + options.mutate?.(core); + core.setCheckpointPersistence(nativeRunnerCheckpointWriter(fixture.db, { + companyId, issueId, runId, authorityRunId: runId, bootId, leaseOwner, generation: 1, + })); + await core.flushCheckpoint(); + } finally { + await core.stop(); + } + } + // The killed controller can no longer renew its lease. + await fixture.db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }) + .where(eq(nativeRunFinalizations.runId, runId)); + return fixture; + } + + it("restarts a claimed remote bootstrap whose durable checkpoint proves no provider work", async () => { + const fixture = await seedClaimedRemoteBootstrap("PROVEN"); + const dispositions = await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] }); + expect(dispositions).toEqual([expect.objectContaining({ kind: "bootstrap_incomplete", runId: fixture.runId, + providerAttempt: 1, controllerGeneration: 2 })]); + const [coordinator] = await fixture.db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + expect(coordinator).toMatchObject({ phase: "observed", recoveryState: "bootstrap_incomplete", failureCode: null }); + expect(coordinator.recoveryHistory.at(-1)).toMatchObject({ disposition: "bootstrap_incomplete", + reason: "remote_bootstrap_proven_incomplete" }); + const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); + expect(run).toMatchObject({ status: "running", errorCode: null, processPid: null }); + expect(await fixture.db.select().from(issueRecoveryActions) + .where(eq(issueRecoveryActions.sourceIssueId, fixture.issueId))).toHaveLength(0); + }); + + it("lets the executor prove the remote bootstrap from its Postgres checkpoint, not the local marker", async () => { + const fixture = await seedClaimedRemoteBootstrap("EXECUTOR"); + const execution = buildNativeExecutionInput({ + companyId, agentId, runId: fixture.runId, + issue: { id: fixture.issueId, identifier: null, title: "Restart bootstrap", description: null, workMode: "standard" }, + taskPrompt: "Restart bootstrap", provider: "codex", + workspace: { id: randomUUID(), cwd: runtimeRoot, repoUrl: null, repoRef: null, branchName: null }, + normalizedSessionId: fixture.native.normalizedSessionId, + runtimeContext: nativeRuntimeContextFixture(), + completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1", + contract: { revision: "1", objective: "Restart bootstrap", criteria: [{ id: "result", requirement: "Complete work" }] } }, + }); + const root = resolve(runtimeRoot, `marker-${fixture.runId}`); + await mkdir(resolve(root, "control-plane"), { recursive: true, mode: 0o700 }); + const writeMarker = (runId: string) => writeFile(resolve(root, "control-plane", "control-plane-state.json"), JSON.stringify({ + schema: "paperclip.runner.external-control-plane-state.v1", + identity: { runnerInstanceId: fixture.native.runnerInstanceId, environmentLeaseId: fixture.runId, runId, + normalizedSessionId: fixture.native.normalizedSessionId, turnId: `turn-${runId}`, itemId: `item-${runId}` }, + })); + const claim = { kind: "bootstrap_incomplete" as const, runId: fixture.runId, leaseOwner: "successor", + controllerGeneration: 2, providerAttempt: 1, restartKind: "hard" as const, recoveryRequestId: null }; + const target = { kind: "remote", transport: "sandbox", remoteCwd: "/workspace" } as AdapterExecutionTarget; + try { + await writeMarker(fixture.runId); + expect(runnerdStateProvesIncompleteBootstrap(root)).toBe(false); + // Only a bootstrap recovery on a classic sandbox restores the checkpoint. + for (const other of [{ restartRecovery: undefined, runnerExecutionTarget: target }, + { restartRecovery: claim, runnerExecutionTarget: { ...target, runnerTask: vi.fn() } as AdapterExecutionTarget }]) { + await restoreRemoteBootstrapCheckpointForRecovery({ db: fixture.db, execution, root, ...other }); + expect(runnerdStateProvesIncompleteBootstrap(root)).toBe(false); + } + // A marker for another run never receives this run's checkpoint. + await writeMarker(randomUUID()); + await restoreRemoteBootstrapCheckpointForRecovery({ db: fixture.db, execution, root, + restartRecovery: claim, runnerExecutionTarget: target }); + expect(runnerdStateProvesIncompleteBootstrap(root)).toBe(false); + await writeMarker(fixture.runId); + await restoreRemoteBootstrapCheckpointForRecovery({ db: fixture.db, execution, root, + restartRecovery: claim, runnerExecutionTarget: target }); + expect(runnerdStateProvesIncompleteBootstrap(root)).toBe(true); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it.each<[string, Parameters[1], string]>([ + ["a turn was issued", { mutate: core => { core.queueCommand("turn.start", { prompt: "Prior work must stay held." }); } }, + "ambiguous_remote_recovery_evidence"], + ["an event was acknowledged", { mutate: core => { core.store.state.ackedSourceSeq = 1; } }, + "ambiguous_remote_recovery_evidence"], + ["the state was adopted", { mutate: core => { core.store.state.bootstrapHistory!.adopted = true; } }, + "ambiguous_remote_recovery_evidence"], + ["no checkpoint exists", { checkpoint: false }, "ambiguous_remote_recovery_evidence"], + ["the bootstrap budget is spent", { attempt: 3 }, "execution_recovery_budget_exhausted"], + ])("keeps a claimed remote bootstrap held when %s", async (label, options, reason) => { + const fixture = await seedClaimedRemoteBootstrap(label.replace(/\W+/g, "-"), options); + expect(await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "hard", runIds: [fixture.runId] })).toEqual([{ kind: "blocked", runId: fixture.runId, reason }]); + }); + it.each([ { attempt: 2, driverKind: "codex_app_server" }, { attempt: 3, driverKind: "codex_app_server" }, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 5d2321c14e6..c7fddc8388b 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -85,6 +85,8 @@ import { createNativeToolTrace, type NativeToolTrace } from "./native-tool-trace import { createNativeProviderFailureObservation } from "./native-provider-failure.js"; import { nativeStartupRetryExhaustedAction, nativeStartupRetryWait, planNativeStartupRetry } from "./native-startup-retry.js"; import { admitUnclaimedNativeBootstrapRetry } from "./native-bootstrap-retry.js"; +import { RUNNERD_CONTROL_PLANE_STATE_SCHEMAS, runnerdStateIsIncompleteBootstrap, + runnerdStateIsUnconnectedBootstrap } from "./native-bootstrap-proof.js"; import { createNativeDeliverableTextObservation } from "./native-deliverable-text.js"; import { createNativeGitHubAccess, type NativeGitHubAccess } from "./native-github-access.js"; import { resolveGitHubOperationCredentials } from "../github-operation-credentials.js"; @@ -522,10 +524,6 @@ const ACPX_RUNTIME_CHECKPOINT_EXCLUDED_ENTRIES = [ "home/.npm", "data/pnpm/store", ] as const; -const RUNNERD_CONTROL_PLANE_STATE_SCHEMAS = new Set([ - "paperclip.runner.durable.control-plane-state.v1", - "paperclip.runner.durable.control-plane-state.v2", -]); const RUNNERD_STATE_SCHEMA = "paperclip.runner.durable.state.v1"; const CODEX_PROVIDER_STATE_SCHEMA = "paperclip.runner.codex-provider-state.v1"; const ACPX_PROVIDER_STATE_SCHEMA = "paperclip.runner.acpx-provider-state.v3"; @@ -5486,6 +5484,7 @@ async function migrateRunnerdStateRootForExecution(input: { if (input.restartRecovery?.kind === "reattach_remote_runner" && !existsSync(scoped)) { throw new Error("runner_state_identity_mismatch: remote_reattach_root_missing"); } + await restoreRemoteBootstrapCheckpointForRecovery({ ...input, root: scoped }); if (existsSync(scoped)) { if (!isSafeNativeStateDirectory(scoped)) { throw new Error("runner_state_directory_unsafe"); @@ -5952,55 +5951,6 @@ function localProcessDefinitelyGone(value: unknown, group = false): boolean { } } -const RUNNERD_BOOTSTRAP_COMMAND_TYPES = new Set(["run.prepare", "session.open"]); - -/** The runner never connected, so every bootstrap command is still undelivered. */ -function runnerdStateIsUnconnectedBootstrap(state: Record): boolean { - const commands = Array.isArray(state.commands) ? state.commands.map(record) : []; - const committedEvents = Array.isArray(state.committedEvents) ? state.committedEvents : []; - return RUNNERD_CONTROL_PLANE_STATE_SCHEMAS.has(String(state.schema)) && - state.connectionCount === 0 && committedEvents.length === 0 && - commands.every(command => command.status === "pending" && - RUNNERD_BOOTSTRAP_COMMAND_TYPES.has(String(command.type))); -} - -/** - * A provider turn can start only after session.open completes and the - * controller issues a turn command. A root whose controller journal stops - * before that point (run.prepare may be acknowledged, session.open never is, - * no event was committed or acknowledged) and whose runner journal emitted no - * event either holds no provider work, so restart recovery may quarantine it - * and bootstrap again. Any other shape stays fail-closed. - */ -function runnerdStateIsIncompleteBootstrap( - state: Record, - runnerJournal?: Record | null, -): boolean { - if (runnerdStateIsUnconnectedBootstrap(state)) return true; - if (!RUNNERD_CONTROL_PLANE_STATE_SCHEMAS.has(String(state.schema))) return false; - if (typeof state.connectionCount !== "number" || !Number.isSafeInteger(state.connectionCount) || - state.connectionCount < 0) return false; - if (!Array.isArray(state.committedEvents) || state.committedEvents.length !== 0) return false; - if (state.ackedSourceSeq !== 0) return false; - if (state.warmTransition != null || state.completedWarmTransition != null || - state.stagedRunAttachment != null) return false; - if (record(state.bootstrapHistory).adopted === true) return false; - if (!Array.isArray(state.commands)) return false; - const commands = state.commands.map(record); - if (!commands.every(command => RUNNERD_BOOTSTRAP_COMMAND_TYPES.has(String(command.type)) && - (command.status === "pending" || - (command.type === "run.prepare" && command.status === "completed")))) return false; - // The controller cannot see runner events that were emitted but never - // committed. When the runner journal exists it must agree that none were. - if (runnerJournal === undefined) return true; - if (runnerJournal === null) return false; - const processed = Object.values(record(runnerJournal.processedCommands)).map(record); - return Array.isArray(runnerJournal.outbox) && runnerJournal.outbox.length === 0 && - runnerJournal.nextSourceSeq === 1 && runnerJournal.ackedSourceSeq === 0 && - processed.every(command => RUNNERD_BOOTSTRAP_COMMAND_TYPES.has(String(command.commandType)) && - (command.commandType === "run.prepare" || command.status !== "completed")); -} - /** `undefined` when the runner never journaled, `null` when unreadable. */ function readRunnerdBootstrapJournal(root: string): Record | null | undefined { const runnerRoot = resolve(root, "runner"); @@ -6029,6 +5979,26 @@ function readRunnerdControlPlaneState(root: string): Record { ); } +/** + * A remote runner's controller state lives in Postgres; its local root holds + * only a fail-closed marker. Before a `bootstrap_incomplete` recovery checks + * that root, restore the exact checkpoint so the same incomplete-bootstrap + * proof applies. A missing checkpoint leaves the marker, which stays fail-closed. + */ +export async function restoreRemoteBootstrapCheckpointForRecovery(input: { + db: Db; execution: NativeExecutionInput; root: string; + restartRecovery?: NativeRestartRecoveryClaim; + runnerExecutionTarget?: AdapterExecutionTarget | null; +}): Promise { + const target = input.runnerExecutionTarget; + if (input.restartRecovery?.kind !== "bootstrap_incomplete" || target?.kind !== "remote" + || target.transport !== "sandbox" || target.runnerTask || !existsSync(input.root) + || readRunnerdDurableIdentity(input.root) !== null) return; + const marker = readRunnerdDurableIdentity(input.root, true); + if (!marker || marker.runId !== input.execution.binding.runId) return; + await restoreNativeRunnerCheckpoint(input.db, input.execution.binding, input.root, nativeSessionKey(input.execution)); +} + export function runnerdStateProvesIncompleteBootstrap(root: string): boolean { try { return runnerdStateIsIncompleteBootstrap(readRunnerdControlPlaneState(root), readRunnerdBootstrapJournal(root));