diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 67d40550fab..aaf5361224e 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3258,6 +3258,26 @@ admission applies the same checks as for a per-turn owner. The journal of a warm owner has no `runner.suspend` command. A local warm Runner and a resumed sandbox that still holds the Runner keep the live adoption path. +A remote Runner whose sandbox lease for the prior run ended (`released` or +`expired`, with a release time) on a different sandbox than this run's is gone +by record. No process remains that could act on a command it never +acknowledged. For such an owner, these unfinished commands do not hold the +retirement: + +- its own lifecycle commands (`runner.suspend`, `runner.drain`, + `runner.shutdown`); +- deliveries that only that Runner could receive: `run.attach`, + `semantic_tool.result`, `session.snapshot` and `session.goal.get`. A + journaled `semantic_tool.result` also answers its tool input, because the + controller journals the answer after the tool ran. + +Retirement replays none of these commands. Any other unfinished command, such +as `turn.start`, `turn.steer`, `request.resolve` or +`external_provider.operation`, still holds the owner. A tool input with no +journaled answer still holds it with `journal_tool_input_unanswered`. A +retained lease, a lease on this run's sandbox, or a missing lease record keeps +every unfinished command decisive. + The provider session then follows the normal resume-or-replace policy. A resumed sandbox, a local warm session, a plugin task, a restart claim, or any missing proof keeps the existing `prior_owner_terminal_state_indeterminate` diff --git a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts index 271da7146e4..15562311197 100644 --- a/server/src/services/native-runtime/native-recovered-owner-followup.test.ts +++ b/server/src/services/native-runtime/native-recovered-owner-followup.test.ts @@ -92,7 +92,7 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered * Only the database controller checkpoint and recovery pointer survive. */ async function seedRecoveredPriorRun(control: "settled" | "pending-command" | "unanswered-tool" | "failed-turn-start" | "rejected-turn-start" | "indeterminate-turn-start" | "unknown-outcome-turn-start" - | "unsuperseded-unknown-outcome-turn-start" | "failed-suspend" = "settled", + | "unsuperseded-unknown-outcome-turn-start" | "failed-suspend" | "pending-attach" | "pending-tool-result" = "settled", options: { controller?: { bootId: string; pid: number; processStartedAt: Date }; provider?: "codex" | "acpx" | "opencode"; lifecycle?: "warm" } = {}) { const issueId = randomUUID(), workspaceId = randomUUID(), runId = randomUUID(); @@ -150,10 +150,16 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered Object.assign(commands.find(command => command.type === "turn.start")!, unknownOutcome); } if (control === "failed-suspend") Object.assign(commands.at(-1)!, { status: "failed", result: { status: "failed" } }); + // Staging PAP-4196 and PAP-3443: the controller journaled a delivery to the + // per-turn Runner that the Runner never acknowledged before its sandbox was + // destroyed after the turn. + if (control === "pending-attach") controller.queueCommand("run.attach", { provider: "fixture" }); + const toolInput = control === "unanswered-tool" || control === "pending-tool-result"; + if (control === "pending-tool-result") controller.queueCommand("semantic_tool.result", { sourceEventId: "unanswered-tool-input" }); const event = (sourceSeq: number, sourceEventId: string, eventType: string) => ({ sourceSeq, sourceEventId, eventType, priority: 0 as const, envelope: { ...identity }, deliveryCount: 1, logicalEffectCount: 1 }); controller.store.state.committedEvents.push( - ...(control === "unanswered-tool" ? [event(1, "unanswered-tool-input", "semantic_tool.input")] : []), + ...(toolInput ? [event(1, "unanswered-tool-input", "semantic_tool.input")] : []), event(2, "recovered-terminal", "run.terminal")); controller.store.state.ackedSourceSeq = 2; controller.setCheckpointPersistence(nativeRunnerCheckpointWriter(db, owner)); @@ -760,6 +766,38 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered expect(await followUp.start()).toBe(outcome); }, 60_000); + // Staging PAP-4196 (run.attach) and PAP-3443 (semantic_tool.result): the + // per-turn owner committed its result and its sandbox was destroyed after the + // turn, but its journal still held a delivery that its Runner never + // acknowledged. No process remains to receive it, so the follow-up retires + // the owner. A Runner whose sandbox may still exist keeps the owner held. + it.each([ + { control: "pending-attach", label: "ended on another sandbox", status: "expired", sameSandbox: false, recorded: true, + outcome: ADMITTED }, + { control: "pending-tool-result", label: "ended on another sandbox", status: "expired", sameSandbox: false, recorded: true, + outcome: ADMITTED }, + { control: "pending-attach", label: "still retained", status: "retained", sameSandbox: false, recorded: true, + outcome: stranded("journal_commands_unsettled(run.attach.pending)") }, + { control: "pending-tool-result", label: "ended on this sandbox", status: "expired", sameSandbox: true, recorded: true, + outcome: stranded("journal_commands_unsettled(semantic_tool.result.pending)") }, + { control: "pending-tool-result", label: "not recorded", status: "expired", sameSandbox: false, recorded: false, + outcome: stranded("journal_commands_unsettled(semantic_tool.result.pending)") }, + { control: "unanswered-tool", label: "ended on another sandbox", status: "expired", sameSandbox: false, recorded: true, + outcome: stranded("journal_tool_input_unanswered") }, + ] as const)("retires a finished remote owner with an unacknowledged $control when its sandbox lease $label", + async ({ control, status, sameSandbox, recorded, outcome }) => { + const prior = await seedRecoveredPriorRun(control, { provider: "opencode" }); + const followUp = await admitFollowUp(prior); + if (recorded) { + await db.insert(environments).values({ id: followUp.environmentId, name: `ended-${followUp.environmentId}`, driver: "sandbox" }); + await db.insert(environmentLeases).values({ companyId, environmentId: followUp.environmentId, issueId: prior.issueId, + heartbeatRunId: prior.runId, provider: "daytona", providerLeaseId: sameSandbox ? followUp.providerLeaseId : `sandbox-${prior.runId}`, + status, releasedAt: status === "retained" ? null : new Date(), metadata: { remoteCwd: "/workspace" } }); + } + expect(await followUp.start()).toBe(outcome); + if (outcome === ADMITTED) expect(followUp.logs.join("")).toContain("finished without a saved runner checkpoint"); + }, 60_000); + it("admits again when a retry rebuilds the same projection from the database", async () => { const prior = await seedRecoveredPriorRun(); const followUp = await admitFollowUp(prior); @@ -1072,6 +1110,17 @@ describe.skipIf(!recoveryPostgresSupport.supported)("follow-up after a recovered expect(await followUp.startLocal()).toBe(stranded("journal_commands_unsettled(turn.start.pending)")); }, 60_000); + it("keeps a stopped local owner held by an unacknowledged run.attach", async () => { + // Only a remote Runner whose sandbox lease ended elsewhere is gone by record. + const { followUp, root } = await seedUnsuspendedLocalOwner(await stoppedRunnerProcess()); + const path = join(root, "control-plane", "control-plane-state.json"); + const state = JSON.parse(await readFile(path, "utf8")); + state.commands.push({ ...state.commands.at(-1), commandId: "command_unacknowledged_attach", + controllerSeq: state.commands.length + 1, type: "run.attach", status: "pending", result: null }); + await writeFile(path, JSON.stringify(state)); + expect(await followUp.startLocal()).toBe(stranded("journal_commands_unsettled(run.attach.pending)")); + }, 60_000); + it.each(["uncommitted result", "committed without result", "live controller lease", "unexpired controller lease", "recovery pending", "pending command", "unanswered tool input", "foreign quarantine"] as const)("keeps the prior owner held with %s", async scenario => { const prior = await seedRecoveredPriorRun(scenario === "pending command" ? "pending-command" diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index dafc06da83e..37a7a8631d2 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -5668,6 +5668,12 @@ function moveNativeContinuityState(root: string, archiveRoot: string): void { type SettledRunnerdProjection = { bytes: string; refused?: never } | { bytes?: never; refused: string }; const LOCAL_RUNNER_LIFECYCLE_COMMANDS = new Set(["runner.suspend", "runner.drain", "runner.shutdown"]); +/** Commands that only deliver something to the Runner that held this journal: + * its attachment, an answer to a tool it already called, or a read. None + * carries new user intent, and none can run without that Runner. When its + * sandbox lease ended on another sandbox, no process remains to receive them. */ +const GONE_REMOTE_RUNNER_DELIVERY_COMMANDS = new Set(["run.attach", "semantic_tool.result", "session.snapshot", + "session.goal.get"]); function readSettledRunnerdProjection(root: string, identity: Record, options: { localRunner?: boolean; controlBytes?: string; remoteRunnerGone?: boolean } = {}): SettledRunnerdProjection { @@ -5690,12 +5696,18 @@ function readSettledRunnerdProjection(root: string, identity: Record !LOCAL_RUNNER_LIFECYCLE_COMMANDS.has(String(command.type)) || command.status === "completed") + ? commands.filter(command => !goneCommandTypes.has(String(command.type)) || command.status === "completed") : commands; const settledControl = settledCommands === commands ? control : { ...control, commands: settledCommands }; + // The controller journals a tool's answer after the tool ran. Only its + // delivery can be outstanding, and only to a Runner that is gone. const answered = new Set(commands - .filter(command => command.type === "semantic_tool.result" && command.status === "completed") + .filter(command => command.type === "semantic_tool.result" && (command.status === "completed" || options.remoteRunnerGone)) .map(command => record(command.payload).sourceEventId)); if (!RUNNERD_CONTROL_PLANE_STATE_SCHEMAS.has(String(control.schema))) return refuse("journal_schema_unsupported"); if (control.warmTransition != null) return refuse("journal_warm_transition");