diff --git a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts index 009f20b6afd..40520c67178 100644 --- a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts +++ b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts @@ -39,7 +39,8 @@ vi.mock("./current-wake-comments.js", async importOriginal => ({ resolveCurrentWakeCommentsBinding: async () => null, assertCurrentWakeCommentsRead: async () => undefined, })); -import { buildNativeHarnessBackupManifest, createRunnerdBackend, resolveNativeHarnessPersistenceProfile, scopedRunnerdStateRoot } from "./native-session-executor.js"; +import { buildNativeHarnessBackupManifest, createRunnerdBackend, quarantinedBootstrapOfOwnerHeldNoProviderWork, resolveNativeHarnessPersistenceProfile, + scopedRunnerdStateRoot } from "./native-session-executor.js"; // Durable-root admission is the boundary under test. Reaching the run's tool // authority proves admission passed; no provider or runner process starts. @@ -648,6 +649,34 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered expect(await quarantined()).toHaveLength(1); }, 60_000); + it.each([ + { label: "an interrupted bootstrap that never opened its session", turnIssued: false, sessionOpened: false }, + { label: "a remote bootstrap that opened its session without a turn", turnIssued: false, sessionOpened: true }, + { label: "a bootstrap copy that already issued a turn", turnIssued: true, sessionOpened: false }, + ])("retires a finished owner past its own quarantined earlier bootstrap: $label", async ({ turnIssued, sessionOpened }) => { + const prior = await seedRecoveredPriorRun(); + const followUp = await admitFollowUp(prior); + await db.update(nativeRunFinalizations).set({ phase: "observed" }).where(eq(nativeRunFinalizations.runId, prior.runId)); + expect(await followUp.start()).toBe(stranded("owner_observed")); + const [copy] = await quarantined(); + const copyRoot = join(stateDirectory, "quarantine", copy!); + const path = join(copyRoot, "control-plane", "control-plane-state.json"); + const state = JSON.parse(await readFile(path, "utf8")); + const bootstrap = state.commands.filter((command: { type: string }) => command.type === "run.prepare" || command.type === "session.open" + || (turnIssued && command.type === "turn.start")); + for (const command of bootstrap) { + command.status = command.type === "run.prepare" || (sessionOpened && command.type === "session.open") ? "completed" : "pending"; + } + const opened = sessionOpened ? ["session.started", "session.capabilities.updated"].map((eventType, index) => ({ + sourceSeq: index + 1, sourceEventId: `opened-${index + 1}`, eventType, priority: 1, envelope: { ...prior.identity }, + deliveryCount: 1, logicalEffectCount: 1 })) : []; + await writeFile(path, JSON.stringify({ ...state, commands: bootstrap, committedEvents: opened, ackedSourceSeq: opened.length })); + await rm(join(copyRoot, "runner"), { recursive: true, force: true }); + await db.update(nativeRunFinalizations).set({ phase: "committed" }).where(eq(nativeRunFinalizations.runId, prior.runId)); + expect(await followUp.start()).toBe(turnIssued + ? stranded("quarantine_journal_commands_unsettled(session.open.pending,turn.start.pending)") : ADMITTED); + }, 60_000); + it("keeps an unsuspended Runner receipt decisive after a retry rebuilds the root", async () => { const prior = await seedRecoveredPriorRun(); const followUp = await admitFollowUp(prior); @@ -770,6 +799,37 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered return { prior, followUp, root }; } + it.each(["exited", "alive", "another owner"] as const)( + "skips a local quarantined bootstrap that opened a session only when it is this owner's and its processes %s", + async scenario => { + const runner = scenario === "alive" ? await runnerProcess() : await stoppedRunnerProcess(); + const root = await mkdtemp(join(tmpdir(), "quarantined-opened-bootstrap-")); + const identity = { runnerInstanceId: "runner-1", environmentLeaseId: "lease-1", runId: "owner-run", + normalizedSessionId: "session-1", turnId: "turn-owner-run", itemId: "item-owner-run" }; + try { + await mkdir(join(root, "control-plane"), { recursive: true, mode: 0o700 }); + await mkdir(join(root, "runner"), { recursive: true, mode: 0o700 }); + await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify({ + schema: "paperclip.runner.durable.control-plane-state.v2", identity, + bootstrapHistory: { schema: "paperclip.runner.bootstrap-history.v1", originIdentity: identity, credentialsIssued: 1, adopted: false }, + commands: [ + { commandId: "prepare", type: "run.prepare", status: "completed", result: { status: "prepared" } }, + { commandId: "open", type: "session.open", status: "completed", result: { result: { status: "started", processId: runner.pid } } }, + ], + committedEvents: [{ sourceSeq: 1, sourceEventId: "opened-1", eventType: "session.started", priority: 1, envelope: identity }], + ackedSourceSeq: 1, connectionCount: 1 })); + await writeFile(join(root, "runner", "runner-state.json"), JSON.stringify({ schema: "paperclip.runner.durable.state.v1", + ...identity, lifecycle: "ready", outbox: [], processedCommands: { + prepare: { commandType: "run.prepare", status: "completed", result: { status: "prepared" } }, + open: { commandType: "session.open", status: "completed", result: { status: "started", processId: runner.pid } } } })); + const owner = scenario === "another owner" ? { ...identity, runId: "another-run" } : identity; + expect(quarantinedBootstrapOfOwnerHeldNoProviderWork(root, owner, true)).toBe(scenario === "exited"); + } finally { + if (scenario === "alive") runner.child.kill("SIGKILL"); + await rm(root, { recursive: true, force: true }); + } + }, 60_000); + it("retires a finished local owner whose Runner stopped before it proved suspension", async () => { const { prior, followUp, root } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess()); expect(await followUp.startLocal()).toBe(ADMITTED); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index bdc1a2c5b92..42d038a2508 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -5612,6 +5612,7 @@ async function inspectFinishedPerTurnRunnerdOwner(input: { if (!isSafeNativeStateDirectory(quarantine)) return refuse("quarantine_unsafe"); for (const name of readdirSync(quarantine)) { if (!name.startsWith(`${basename(input.root)}.`)) continue; + if (quarantinedBootstrapOfOwnerHeldNoProviderWork(resolve(quarantine, name), identity, localRunner)) continue; const copy = readSettledRunnerdProjection(resolve(quarantine, name), identity, { localRunner }); if (copy.refused !== undefined) return refuse(`quarantine_${copy.refused}`); } @@ -5627,6 +5628,15 @@ async function inspectFinishedPerTurnRunnerdOwner(input: { return { finished: true, run: prior.run, recheck }; } +export function quarantinedBootstrapOfOwnerHeldNoProviderWork(root: string, identity: Record, + localRunner: boolean): boolean { + if (!isSafeNativeStateDirectory(root)) return false; + const copied = readRunnerdDurableIdentity(root); + if (!copied || Object.entries(identity).some(([key, value]) => copied[key] !== value)) return false; + return runnerdStateProvesIncompleteBootstrap(root, { localProcesses: localRunner }) + || (!localRunner && runnerdStateProvesNoRemoteProviderTurn(root)); +} + async function retireFinishedPerTurnRunnerdOwner(input: Parameters[0]) : Promise<{ retired: true } | { retired: false; refused: string }> { const proof = await inspectFinishedPerTurnRunnerdOwner(input);