diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index 26f97cf0568..ba8faa2826b 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -2423,6 +2423,13 @@ contract, setup, recovery, boundaries, and qualification status. ### Native task completion +A durable question or approval handoff survives a missing provider shutdown +notification. Paperclip settles its yielded result once the provider terminal +or exact host-owned shutdown proves stopped execution and settled tool outcomes. +Until then it retains the saved handoff and explains the shutdown hold. Recovery +must not repeat completed work or release an overlapping execution. See +`doc/execution-semantics.md` for the bounded settlement and restart contract. + For ordinary low-risk tasks, accept the current agent's structured `done` claim subject to explicit workflow constraints. Missing independent evidence or a `needs_review` label alone must not create a human approval. Require a concrete diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 3ab9a5f8091..fc2058fd367 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -4101,13 +4101,33 @@ run's wait marker along with its terminal, result and pending-request authority. Already-numbered cancellation acknowledgements and cancelled/expired runtime requests are retained during shutdown so recovery can replay a contiguous journal. They carry no permission to start work. Stream closure, stream failure or drain timeout -after complete usage leaves the saved wait unfinalized until a provider terminal -is retained; complete token counters alone cannot certify a stopped provider. -Codex governed-wait settlement allows up to 30 seconds for its interruption -terminal, matching the transport command window. The caller's execution deadline -can shorten that window. An interrupt acknowledgement alone is not terminal proof. -A timeout, missing accounting, or an unproven provider terminal remains a failure; -the saved wait does not certify successful provider execution. +after complete usage leaves the saved wait unfinalized until execution is proven +stopped; complete token counters alone cannot certify that boundary. Codex allows +up to 30 seconds for its interruption terminal, matching the transport command +window; the caller's execution deadline can shorten it. After that bounded drain, +a supporting driver uses its existing owned shutdown. +For Runner, this requires exact run/session/turn identity, provider settlement, +durable event drain, settled semantic tool outcomes and verified Runner +suspension. An interrupt acknowledgement, process signal or ordinary close +promise alone is insufficient. This proof settles the saved yielded result +without asserting provider success or complete accounting. The stopped session +is retired from warm reuse. + +The checkpoint may add a shutdown settlement receipt to the existing governed +wait. Previous checkpoints without it follow the same terminal-or-shutdown +settlement path. Recovery validates the original triggering event and receipt +binding, rejects later turns and failed provider terminals, and completes the +same saved handoff without starting a model turn, even if its question has been +answered. A crash after receipt persistence retains that proof. Late settlement +after controller cancellation cannot publish a result. + +If shutdown cannot be proven, the run retains its checkpoint and the named +`native_governed_wait_settlement_pending` condition. The existing bounded +same-run recovery schedule owns further settlement; it grants no replacement +or tool replay. The task explains that work is saved and shutdown is pending. +Exhausted or gated recovery holds continuation and names the shutdown problem. +Actual provider failures and integrity failures retain their existing meaning. +Accounting keeps its separate receipt and reconciliation requirements. An exact governed-cancellation terminal with an explicit turn ID may return its durable event receipt before postcommit accounting and output observers finish. diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index a8caaa4a643..dfaee9efff3 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -776,6 +776,12 @@ class HarnessNativeSession implements NativeSession { this.#explicitlyCancelled = true; } + settleCancelledTurn(input: { turnId: string; reason: string }) { + this.#assertProtocolIntegrity(); + if (!this.#explicitlyCancelled) throw new Error("native_turn_cancellation_required"); + return this.#withProtocolIntegrity(() => this.#session.settleCancelledTurn?.(input) ?? null); + } + cancel(input: { reason: string; signal: AbortSignal }) { if (input.signal.aborted) { throw ( diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index d4ccd77bc67..7be7eccfc9a 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -555,6 +555,12 @@ export function runnerRecoveryFailureClass( return runnerRecoveryFailureClass(value.cause, depth + 1); } +export interface CancelledTurnSettlement { + runId: string; + normalizedSessionId: string; + turnId: string; +} + export interface HarnessSession { supportsTurnReasoning?(): boolean; turnControlCapabilities?(): NativeTurnControlCapabilities | null; @@ -602,6 +608,7 @@ export interface HarnessSession { reason?: string; signal?: AbortSignal; }): Promise; + settleCancelledTurn?(input: { turnId: string; reason: string }): Promise; pendingRuntimeRequests?(): HarnessRuntimeRequest[]; resolveRuntimeRequest?(input: { /** Durable controller command identity, distinct from the native request identity. */ diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index 282d74a83bc..e76ea62817a 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -23,6 +23,7 @@ import type { PersistedHarnessProviderIdentity, PersistedHarnessSession, PersistedHarnessTurnTerminal, + CancelledTurnSettlement, } from "./harness-driver.js"; export interface NativeSessionBackendDescriptor { @@ -70,7 +71,7 @@ export interface PersistedNativeSession { terminalTurns?: PersistedHarnessTurnTerminal[]; /** Control-plane disposition bound to the exact committed tool event. It is * independent of the interaction's later answer and proves no provider terminal. */ - governedWait?: { sourceEvent: PrpEvent; result: PrpStructuredRunResult }; + governedWait?: { sourceEvent: PrpEvent; result: PrpStructuredRunResult; settlement?: CancelledTurnSettlement }; /** At-most-once marker for resultless disposition repair or restart continuation. */ dispositionOnlyRecoveryConsumed?: boolean; @@ -266,6 +267,7 @@ export interface NativeSession { reason: string; signal: AbortSignal; }): NativeSessionCancellation; + settleCancelledTurn?(input: { turnId: string; reason: string }): Promise; resolveRuntimeRequest?(input: { commandId?: string; requestId: string; @@ -344,6 +346,14 @@ export class NativeProviderSideEffectPendingError extends Error { } } +export class NativeGovernedWaitSettlementPendingError extends Error { + readonly code = "native_governed_wait_settlement_pending"; + constructor(options?: ErrorOptions) { + super("The waiting state is saved; confirming shutdown of the previous execution is still pending.", options); + this.name = "NativeGovernedWaitSettlementPendingError"; + } +} + /** A provider failed terminal is not a missing completion proposal. */ export class NativeProviderTerminalFailure extends Error { readonly code = "native_provider_terminal_failed"; diff --git a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts index cbb6186a008..ff0affb416f 100644 --- a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts +++ b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts @@ -3,7 +3,7 @@ import type { DurableRecoveryIdentity } from "../../contracts/durable-recovery.j import type { NativeTurnControlCapabilities } from "../../contracts/types.js"; import type { NativeRunRuntimeGrant } from "../../contracts/runtime-context.js"; import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; -import type { HarnessRuntimeRequestResolution } from "../../contracts/harness-driver.js"; +import type { HarnessRuntimeRequestResolution, HarnessSession } from "../../contracts/harness-driver.js"; import { githubCredentialEnvironment } from "../../github-credential-environment.js"; import { redactCodexDiagnostic } from "./diagnostic-redaction.js"; @@ -37,6 +37,7 @@ export type CodexServerRequestHandler = ( ) => Promise>; export interface CodexAppServerTransport { + settleCancelledTurn?: HarnessSession["settleCancelledTurn"]; supportsTurnReasoning?(): boolean; /** Runner-owned live capability projection; absent on native Codex transports. */ turnControlCapabilities?(): NativeTurnControlCapabilities | null; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts index a3e0a3c0e04..6de9c319e24 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts @@ -919,6 +919,19 @@ export class CodexHarnessSession }; } + async settleCancelledTurn(input: { turnId: string; reason: string }) { + this.assertProtocolIntegrity(); + if (this.activeTurnId !== null && this.activeTurnId !== input.turnId) return null; + const receipt = await this.transport.settleCancelledTurn?.(input) ?? null; + this.assertProtocolIntegrity(); + if (receipt) { + this.cancelPendingRequests("session_closed"); + this.activeTurnId = null; + this.eventQueue.close(); + } + return receipt; + } + // `force` is part of the contract. Codex closes the same way with or without it. async close(input: { reason: string; force?: boolean }): Promise { this.cancelPendingRequests("session_closed"); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index d52e8cca619..6bc1901d05b 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -3998,6 +3998,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { readonly #closedSignal: Promise; #resolveClosedSignal!: () => void; #closePromise: Promise | null = null; + #shutdownSettled = false; #probeRetirement: Promise | null = null; readonly prepareConversationResume?: CodexAppServerTransport["prepareConversationResume"]; #detachPromise: Promise | null = null; @@ -5242,6 +5243,17 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { return this.#closePromise; } + async settleCancelledTurn(input: { turnId: string; reason: string }) { + const core = this.#core; + if (!core || this.#controllerDetachedForRestart || this.#releaseEpisode || this.#turnStartResponsePending + || (this.controllerAcceptedCompletedTurnId() ?? this.#turnId) !== input.turnId) return null; + const identity = structuredClone(core.store.state.identity); + await this.close(input.reason); + if (!this.#shutdownSettled || this.#controllerDetachedForRestart || this.#core !== core + || !recoveryIdentityMatches(core.store.state.identity, identity)) return null; + return { runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, turnId: input.turnId }; + } + retireProbeResources(timeoutMs?: number): Promise { if (this.#probeRetirement) return this.#probeRetirement; const operation = (async () => { @@ -5633,6 +5645,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if (!preparationConfirmed) { throw new Error(`Native runner ${attachmentPending ? "attachment" : "startup"} cleanup is unconfirmed; recovery state retained.`); } + this.#shutdownSettled = runnerSettled; if (this.#ownsRoot && !adoptedRunner) { rmSync(this.#root, { recursive: true, force: true }); } diff --git a/packages/paperclip-runner/src/live/runnerd-governed-wait.test.ts b/packages/paperclip-runner/src/live/runnerd-governed-wait.test.ts index e0413ee00b3..d009204dda3 100644 --- a/packages/paperclip-runner/src/live/runnerd-governed-wait.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-governed-wait.test.ts @@ -10,7 +10,7 @@ import { executeNativeSession } from "../native-session-runtime.js"; import { createRunnerdCodexTransport } from "./runnerd-codex-transport.js"; import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; -it.each([0, 18_000, null])("requires a proven Codex terminal when parking a wait (delay: %s)", async terminalDelayMs => { +it.each([0, 18_000, null])("requires a Codex terminal or verified shutdown when parking a wait (delay: %s)", async terminalDelayMs => { const root = await mkdtemp(join(tmpdir(), "runner-governed-wait-")); const input: NativeExecutionInputV1 = { schema: "paperclip.native-execution-input.v1", @@ -93,13 +93,17 @@ it.each([0, 18_000, null])("requires a proven Codex terminal when parking a wait }, resolveGovernedWait: ({ event }) => event.eventType === "item.completed" && event.payload.kind === "dynamicToolCall" ? yielded : null, }); + await expect(execution).resolves.toMatchObject({ result: yielded }); + expect(completeRun).toHaveBeenCalledOnce(); if (terminalDelayMs === null) { - await expect(execution).rejects.toMatchObject({ providerCode: "governed_wait_provider_terminal_unproven" }); - expect(completeRun).not.toHaveBeenCalled(); - expect(history.some(event => event.eventType === "run.terminal")).toBe(false); + expect(history.filter(event => ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(event.eventType))).toEqual([]); + expect(checkpoint?.governedWait?.settlement).toEqual({ runId: input.binding.runId, + normalizedSessionId: input.session.normalizedSessionId, turnId: checkpoint?.governedWait?.sourceEvent.turnId }); + const journal = JSON.parse(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")); + expect(journal.commands.find((command: { type: string }) => command.type === "turn.stop")) + .toMatchObject({ status: "completed", result: { result: { providerExitConfirmed: true } } }); + expect(JSON.parse(await readFile(join(root, "runner/runner-state.json"), "utf8"))).toMatchObject({ lifecycle: "suspended" }); } else { - await expect(execution).resolves.toMatchObject({ result: yielded }); - expect(completeRun).toHaveBeenCalledOnce(); expect(history.some(event => event.eventType === "turn.interrupted")).toBe(true); } expect(checkpoint?.governedWait?.result).toEqual(yielded); diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index e09f3a35dad..c8386aaf6d5 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -1,7 +1,10 @@ import { validatePrpStructuredRunResult } from "./protocol/replay-contract.js"; import { createProviderFailure } from "./drivers/provider-failure.js"; import { createHash } from "node:crypto"; -import { writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { createCapabilityRunnerdCodexTransport } from "./live/runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../test/support/runnerd-transport.js"; import { join } from "node:path"; import { resolveQualifiedAcpxProfile } from "./drivers/acpx/qualified-profiles.js"; import { describe, expect, it, vi } from "vitest"; @@ -9202,6 +9205,164 @@ describe("executeNativeSession recovery", () => { ]); }); + it("settles a governed wait through real Runner shutdown after a missing Codex terminal", async () => { + const directory = await mkdtemp(join(tmpdir(), "governed-wait-stop-")); + const executionInput = { ...input, workspace: { ...input.workspace, cwd: directory } }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(directory, "--emit-tool-call", "--hold-turn", "--accept-interrupt-without-terminal"), + stateDirectory: directory, lifecyclePolicy: input.session.lifecyclePolicy, + prpIdentity: { runnerInstanceId: "runner-recovery", environmentLeaseId: "lease-recovery", runId: identity.runId, + normalizedSessionId: identity.sessionId, turnId: "turn-bootstrap", itemId: "item-bootstrap" }, + }); + const tool = vi.fn(async () => { + await writeFile(join(directory, "saved-work.txt"), "The work and question were saved."); + return { success: true, contentItems: [{ type: "inputText", text: "Saved question" }] }; + }); + const backend = createRunnerdNativeSessionBackend(executionInput, { + runnerInstanceId: "runner-recovery", transportFactory: () => bundle.transport, + dynamicTools: [{ name: "get_task_context", description: "Persist the fixture handoff", inputSchema: { type: "object", properties: {} } }], + dynamicToolHandler: tool, + }); + const events: PrpEvent[] = []; + let checkpoint: PersistedNativeSession | undefined; + const complete = vi.fn(async () => undefined); + const port: ControlPlanePort = { + async openRun() {}, completeRun: complete, + async checkpointSession(value) { checkpoint = structuredClone(value); }, + async appendEvent(event: PrpEvent) { events.push(event); return { cursor: events.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, + async replayEvents(request) { return { events: events.filter(event => event.sourceInstanceId === request.sourceInstanceId && event.sourceSeq > request.afterSourceSeq), highestContiguousSourceSeq: 0 }; }, + }; + try { + const result = await executeNativeSession({ input: executionInput, backend, controlPlane: port, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 30_000, + requireSessionCloseBeforeReturn: true, + onSession(session) { + if (!session?.cancel) return; + const cancel = session.cancel.bind(session); + session.cancel = request => ({ ...cancel(request), terminalDrainTimeoutMs: 100 }); + }, + resolveGovernedWait: ({ event }) => event.eventType === "item.completed" && event.payload.kind === "dynamicToolCall" + ? yieldedResult : null, + }); + expect(result).toMatchObject({ result: yieldedResult, terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded" } }); + expect(tool).toHaveBeenCalledOnce(); + expect(complete).toHaveBeenCalledOnce(); + expect(await readFile(join(directory, "saved-work.txt"), "utf8")).toBe("The work and question were saved."); + expect(checkpoint?.governedWait?.settlement).toMatchObject({ runId: identity.runId, normalizedSessionId: identity.sessionId }); + expect(events.some(event => ["turn.completed", "turn.failed"].includes(event.eventType))).toBe(false); + const journal = JSON.parse(await readFile(join(directory, "control-plane/control-plane-state.json"), "utf8")); + expect(journal.commands.filter((command: { type: string }) => command.type === "turn.start")).toHaveLength(1); + expect(journal.commands.find((command: { type: string }) => command.type === "turn.stop")).toMatchObject({ + status: "completed", result: { result: { providerExitConfirmed: true } }, + }); + expect(JSON.parse(await readFile(join(directory, "runner/runner-state.json"), "utf8"))).toMatchObject({ lifecycle: "suspended" }); + } finally { + await bundle.transport.close().catch(() => undefined); + await rm(directory, { recursive: true, force: true }); + } + }, 45_000); + + it.each(["confirmed", "restart", "foreign-turn", "no-proof", "stop-failed", "timeout", "provider-failed", "terminal-write-failed"] as const)("settles only verified governed shutdown (%s)", async fault => { + vi.useFakeTimers(); + try { + const trigger = { ...runnerEvent(1, "item.completed", { kind: "dynamicToolCall" }), turnId: "turn-waiting" }; + const history: PrpEvent[] = []; + let checkpoint: PersistedNativeSession | undefined; + let releaseStream!: () => void; + const streamClosed = new Promise(resolve => { releaseStream = resolve; }); + let confirmStop!: () => void; + const stopConfirmed = new Promise(resolve => { confirmStop = resolve; }); + const complete = vi.fn(async () => undefined); + const start = vi.fn(async () => ({ turnId: trigger.turnId })); + const settleCancelledTurn = vi.fn(async () => { + await stopConfirmed; + releaseStream(); + if (fault === "stop-failed") throw new Error("shutdown unconfirmed"); + if (fault === "no-proof") return null; + return { runId: identity.runId, normalizedSessionId: identity.sessionId, + turnId: fault === "foreign-turn" ? "another-turn" : trigger.turnId }; + }); + let crashed = false; + const session: NativeSession & { settleCancelledTurn: typeof settleCancelledTurn } = { + identity: () => identity, + async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, + startTurn: start, + cancel() { return { cleanup: Promise.resolve() }; }, + settleCancelledTurn, + async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, activeTurnId: null }; }, + async result() { return null; }, async close() { releaseStream(); }, + async *events() { + yield trigger; + yield { ...runnerEvent(2, "item.completed", { kind: "interrupt_acknowledgement" }), turnId: trigger.turnId }; + await streamClosed; + if (fault === "provider-failed" || fault === "terminal-write-failed") yield { ...runnerEvent(3, "turn.failed", { error: { code: "provider_failed" } }), turnId: trigger.turnId }; + }, + }; + const backend: NativeSessionBackend = { + async descriptor() { return { kind: "mock", name: "verified-stop", version: "1", capabilities: await session.capabilities() }; }, + async openSession() { return session; }, + async recoverSession() { return { recovered: true, session }; }, + }; + const port: ControlPlanePort = { + async openRun() {}, completeRun: complete, + async checkpointSession(value) { + checkpoint = structuredClone(value); + if (fault === "restart" && value.governedWait?.settlement && !crashed) { + crashed = true; + throw new Error("controller stopped after settlement checkpoint"); + } + }, + async appendEvent(event: PrpEvent) { + if (fault === "terminal-write-failed" && event.eventType === "turn.failed") throw new Error("terminal persistence unavailable"); + history.push(event); return { cursor: history.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; + }, + async replayEvents(request) { return { events: history.filter(row => row.sourceInstanceId === request.sourceInstanceId && row.sourceSeq > request.afterSourceSeq), highestContiguousSourceSeq: 0 }; }, + }; + const execution = executeNativeSession({ input, backend, controlPlane: port, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveGovernedWait: () => yieldedResult }); + void execution.catch(() => undefined); + await vi.advanceTimersByTimeAsync(15_001); + expect(checkpoint?.governedWait).toMatchObject({ sourceEvent: trigger, result: yieldedResult }); + expect(complete).not.toHaveBeenCalled(); + expect(settleCancelledTurn).toHaveBeenCalledOnce(); + if (fault === "timeout") { + await vi.advanceTimersByTimeAsync(30_500); + await expect(execution).rejects.toMatchObject({ code: "native_governed_wait_settlement_pending" }); + confirmStop(); + await vi.advanceTimersByTimeAsync(1); + expect(complete).not.toHaveBeenCalled(); + expect(checkpoint?.governedWait?.settlement).toBeUndefined(); + return; + } + confirmStop(); + await vi.advanceTimersByTimeAsync(1); + if (["foreign-turn", "no-proof", "stop-failed", "provider-failed", "terminal-write-failed"].includes(fault)) { + await expect(execution).rejects.toMatchObject({ code: fault === "foreign-turn" ? "native_event_replay_conflict" + : fault === "provider-failed" ? "native_provider_terminal_failed" : "native_governed_wait_settlement_pending" }); + expect(complete).not.toHaveBeenCalled(); + expect(checkpoint?.governedWait?.settlement).toBeUndefined(); + return; + } + if (fault === "restart") { + await expect(execution).rejects.toThrow("controller stopped after settlement checkpoint"); + expect(complete).not.toHaveBeenCalled(); + const answerAlreadySaved = vi.fn(() => null); + await expect(executeNativeSession({ input, backend, controlPlane: port, persistedSession: checkpoint, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveGovernedWait: answerAlreadySaved })) + .resolves.toMatchObject({ result: yieldedResult }); + expect(answerAlreadySaved).not.toHaveBeenCalled(); + expect(settleCancelledTurn).toHaveBeenCalledOnce(); + } else { + expect(await execution).toMatchObject({ result: yieldedResult, terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded" } }); + } + expect(complete).toHaveBeenCalledOnce(); + expect(start).toHaveBeenCalledOnce(); + expect(history.filter(event => event.eventType === "run.result.accepted")).toHaveLength(1); + expect(history.some(event => event.eventType === "turn.completed")).toBe(false); + } finally { vi.useRealTimers(); } + }); + it.each(["end", "throw", "timeout"])("keeps a governed wait unfinalized when complete usage is followed by %s without a provider terminal", async fault => { vi.useFakeTimers(); try { @@ -9237,7 +9398,7 @@ describe("executeNativeSession recovery", () => { }; const outcome = expect(executeNativeSession({ input, backend, controlPlane: port, timeoutMs: 1_000, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveGovernedWait: () => yieldedResult })) - .rejects.toMatchObject({ providerCode: "governed_wait_provider_terminal_unproven" }); + .rejects.toMatchObject({ code: "native_governed_wait_settlement_pending" }); void outcome.catch(() => undefined); await vi.advanceTimersByTimeAsync(600); await outcome; @@ -9301,7 +9462,7 @@ describe("executeNativeSession recovery", () => { } await vi.advanceTimersByTimeAsync(fault === "abort" ? 1_002 : 2); if (fault === "abort") await expect(execution).rejects.toThrow("controlled startup abort"); - else await expect(execution).rejects.toMatchObject({ code: "native_provider_terminal_failed" }); + else await expect(execution).rejects.toMatchObject({ code: fault === "unproven" ? "native_provider_terminal_failed" : "native_governed_wait_settlement_pending" }); expect(checkpoint?.governedWait).toEqual({ sourceEvent: trigger, result: yieldedResult }); expect(complete).not.toHaveBeenCalled(); expect(history.some(event => event.eventType === "run.terminal")).toBe(false); @@ -9426,11 +9587,14 @@ describe("executeNativeSession recovery", () => { expect(history.filter(event => event.payload.kind === "usage")).toEqual([usage]); }); - it.each(["missing-trigger", "changed-trigger", "unproven-terminal", "different-turn"])("rejects unsafe governed-wait recovery: %s", async fault => { + it.each(["missing-trigger", "changed-trigger", "unproven-terminal", "different-turn", "foreign-settlement", "active-settled-turn"])("rejects unsafe governed-wait recovery: %s", async fault => { const trigger = { ...runnerEvent(1, "item.completed", { kind: "dynamicToolCall" }), turnId: "turn-waiting" }; const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: identity.sessionId, identity, activeTurnId: trigger.turnId, governedWait: { sourceEvent: trigger, result: yieldedResult } }; const history = fault === "missing-trigger" ? [] : [fault === "changed-trigger" ? { ...trigger, payload: {} } : trigger]; + if (fault === "foreign-settlement" || fault === "active-settled-turn") checkpoint.governedWait!.settlement = { + runId: fault === "foreign-settlement" ? "another-run" : identity.runId, normalizedSessionId: identity.sessionId, turnId: trigger.turnId, + }; if (fault === "unproven-terminal") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "unproven", eventType: "turn.failed", payload: { providerTerminalObserved: false } }); if (fault === "different-turn") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "new-turn", turnId: "different", eventType: "turn.started" }); const start = vi.fn(); @@ -9443,7 +9607,7 @@ describe("executeNativeSession recovery", () => { const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { throw new Error("unexpected append"); }, completeRun: complete, async replayEvents(request) { return { events: history.filter(row => row.sourceSeq > request.afterSourceSeq), highestContiguousSourceSeq: history.at(-1)?.sourceSeq ?? 0 }; } }; await expect(executeNativeSession({ input, backend, controlPlane: port, persistedSession: checkpoint, - runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" })).rejects.toThrow(/governed_wait|failed terminal/); + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" })).rejects.toThrow(/governed_wait|failed terminal|native_event_replay_conflict/); expect(start).not.toHaveBeenCalled(); expect(complete).not.toHaveBeenCalled(); }); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 995b143d4f2..433b8fbb8ec 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -29,6 +29,7 @@ import { NativeProviderSideEffectPendingError, NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, + NativeGovernedWaitSettlementPendingError, NativeSessionCloseUnrecoverableError, NativeSessionCleanupQuarantinedError, NativeSessionProtocolIntegrityError, @@ -1231,26 +1232,51 @@ async function consumeTurn( } } })(); - await Promise.race([ - drained.catch(error => { - quarantineSession("governed_accounting_drain_failed"); - if (error instanceof NativeProviderTerminalFailure) throw error; - }), - new Promise(resolve => { drainTimer = setTimeout(() => { - drainAbort.abort(new Error("governed accounting drain timed out")); - quarantineSession("governed_accounting_drain_timed_out"); - resolve(); - }, drainMs); }), - ]).finally(() => { + const observedDrain = drained.catch(error => { + quarantineSession("governed_accounting_drain_failed"); + if (error instanceof NativeProviderTerminalFailure || error instanceof NativeSessionProtocolIntegrityError) throw error; + throw new NativeGovernedWaitSettlementPendingError({ cause: error }); + }); + try { + await Promise.race([ + observedDrain, + new Promise(resolve => { drainTimer = setTimeout(() => { + quarantineSession("governed_accounting_drain_timed_out"); + resolve(); + }, drainMs); }), + ]); + clearTimeout(drainTimer); + if (!governedProviderTerminalObserved) { + const settlement = await runAbortableOperationWithin({ + timeoutMs: MAX_GOVERNED_ACCOUNTING_DRAIN_MS, + timeoutMessage: "governed wait shutdown settlement timed out", + timeoutError: () => new NativeGovernedWaitSettlementPendingError(), + signal: appendAbort.signal, + operation: async signal => { + let receipt; + try { + receipt = await session.settleCancelledTurn?.({ turnId: event.turnId!, reason }); + } catch (error) { + if (error instanceof NativeSessionProtocolIntegrityError) throw error; + throw new NativeGovernedWaitSettlementPendingError({ cause: error }); + } + signal.throwIfAborted(); + if (!receipt) throw new NativeGovernedWaitSettlementPendingError(); + if (receipt.runId !== event.runId || receipt.normalizedSessionId !== event.normalizedSessionId + || receipt.turnId !== event.turnId) throw new NativeSessionProtocolIntegrityError("source_event_replay_conflict"); + await observedDrain; + signal.throwIfAborted(); + return receipt; + }, + }); + await checkpointGovernedWait?.({ sourceEvent: event, result, settlement }, appendAbort.signal); + appendAbort.signal.throwIfAborted(); + quarantineSession("governed_wait_session_retired"); + } + } finally { clearTimeout(drainTimer); appendAbort.signal.removeEventListener("abort", stopDrain); drainAbort.abort(); - }); - // A durable wait and complete usage cannot prove that the provider - // stopped. Leave the journaled wait recoverable, without a successful - // result or an accounting-completion boundary, if shutdown is unproven. - if (!governedProviderTerminalObserved) { - throw new NativeProviderTerminalFailure("governed_wait_provider_terminal_unproven", false); } return { event, @@ -2326,6 +2352,9 @@ export async function executeNativeSession( if (governedReplay.some(entry => entry.eventType === "turn.started" && entry.turnId !== event.turnId)) { throw new Error("native_governed_wait_turn_mismatch"); } + const settlement = governedWait.settlement; + if (settlement && (settlement.runId !== event.runId || settlement.normalizedSessionId !== event.normalizedSessionId + || settlement.turnId !== event.turnId)) throw new NativeSessionProtocolIntegrityError("source_event_replay_conflict"); } let recovered = false; let session: NativeSession | null = null; @@ -2827,6 +2856,9 @@ export async function executeNativeSession( if (governedWait && recoveredActiveTurnId && recoveredActiveTurnId !== governedWait.sourceEvent.turnId) { throw new Error("native_governed_wait_turn_mismatch"); } + if (governedWait?.settlement && recoveredActiveTurnId) { + throw new NativeSessionProtocolIntegrityError("source_event_replay_conflict"); + } if (governedWait) { for (const event of governedReplay) { if (event.turnId === governedWait.sourceEvent.turnId && (event.payload.kind === "usage" || isTurnTerminal(event))) { @@ -2916,9 +2948,9 @@ export async function executeNativeSession( replayedDisposition?.terminal ?? dispositionFallback; const consumptionAbort = new AbortController(); const consuming = - governedTerminal && governedWait + (governedTerminal || governedWait?.settlement) && governedWait ? Promise.resolve({ event: governedWait.sourceEvent, governedResult: governedWait.result, - eventCount: 0, highestContiguousSourceSeq: governedTerminal.sourceSeq }) + eventCount: 0, highestContiguousSourceSeq: governedReplay.at(-1)?.sourceSeq ?? governedWait.sourceEvent.sourceSeq }) : recoveryTerminal === null ? consumeTurn( session, diff --git a/packages/shared/src/types/execution-projection.ts b/packages/shared/src/types/execution-projection.ts index 22fce6bdcd7..72b26df10c0 100644 --- a/packages/shared/src/types/execution-projection.ts +++ b/packages/shared/src/types/execution-projection.ts @@ -68,6 +68,7 @@ export const EXECUTION_RECONCILIATION_CAUSES = [ "provider_failure_meaning_unverified", "provider_ownership_unverified", "native_provider_terminal_failed", + "native_governed_wait_settlement_pending", "native_event_replay_conflict", "native_session_cleanup_quarantined", "native_session_retry_exhausted", diff --git a/server/src/services/execution-projection.test.ts b/server/src/services/execution-projection.test.ts index e5c6eb289ee..e8b4a049e7d 100644 --- a/server/src/services/execution-projection.test.ts +++ b/server/src/services/execution-projection.test.ts @@ -275,6 +275,14 @@ describe("execution truth projection", () => { it("includes the original legacy attempt in the displayed incident budget", () => { expect(projectExecution(run({ runtimeMode: "legacy", status: "scheduled_retry", scheduledRetryAttempt: 1 }), undefined, [], undefined, now)).toMatchObject({ attempt: 2, maxAttempts: 3 }); }); + it("keeps saved governed work in shutdown reconciliation without offering a new attempt", () => { + expect(project( + run({ status: "failed", errorCode: "native_governed_wait_settlement_pending" }), + coordinator({ phase: "retryable_failure", failureCode: "native_governed_wait_settlement_pending", + nextAttemptAt: new Date(now.getTime() + 30_000) }), + )).toMatchObject({ phase: "finishing", label: "Finishing shutdown", permittedActions: ["inspect_run"], + nextAction: "The work and waiting state are saved. Paperclip is confirming shutdown before continuing." }); + }); it("shows pending interaction only after productive work has stopped", () => { expect( project(run(), coordinator(), [{ kind: "connection_intent" }]).phase, diff --git a/server/src/services/execution-projection.ts b/server/src/services/execution-projection.ts index ab3de935360..95a0db338ef 100644 --- a/server/src/services/execution-projection.ts +++ b/server/src/services/execution-projection.ts @@ -402,6 +402,10 @@ export function projectExecution( run.status === "scheduled_retry" )) { projection.recoveryOwner = "agent"; + if (coordinator?.failureCode === "native_governed_wait_settlement_pending") { + projection.nextAction = "The work and waiting state are saved. Paperclip is confirming shutdown before continuing."; + return set("finishing", "Finishing shutdown"); + } return set( projection.retryAt && new Date(projection.retryAt) > now ? "retry_scheduled" diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 01db246fb72..16de3f1308b 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -95,6 +95,7 @@ import { NativeProviderSideEffectPendingError, NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, + NativeGovernedWaitSettlementPendingError, NativeSessionProtocolIntegrityError, } from "../../vendor/paperclip-runner/index.js"; import * as issueServiceModule from "../issues.js"; @@ -20768,6 +20769,19 @@ describe("runnerd provider runtime wiring", () => { }); }); +it("reconciles a saved governed wait with a bounded same-run settlement schedule", () => { + const now = new Date("2026-11-09T12:00:00Z"); + const code = nativeSessionFailureSourceCode(new NativeGovernedWaitSettlementPendingError()); + expect(code).toBe("native_governed_wait_settlement_pending"); + expect(nativeSessionFailureDisposition(1, now, code)).toEqual({ + phase: "retryable_failure", failureCode: code, nextAttemptAt: new Date(now.getTime() + 30_000), + }); + expect(nativeSessionFailureDisposition(3, now, code)).toEqual({ + phase: "terminal_failure", failureCode: code, nextAttemptAt: null, + }); + expect(requiresExecutionReconciliation(code)).toBe(true); +}); + // Terminal wrapping must never turn an authorization/integrity failure into a safe replacement. it.each([ "tool_binding_mismatch", diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index e731e4dda80..e8df4b0b52f 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -175,6 +175,7 @@ import { NativeProviderSideEffectPendingError, NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, + NativeGovernedWaitSettlementPendingError, NativeRunnerSourceReconciliationRequiredError, NativeSessionCleanupQuarantinedError, NativeSessionCloseUnrecoverableError, @@ -8001,7 +8002,7 @@ export function nativeSessionFailureDisposition( phase: exhausted ? ("terminal_failure" as const) : ("retryable_failure" as const), - failureCode: permanentFailure || permissionDeclined + failureCode: permanentFailure || permissionDeclined || sourceFailureCode === "native_governed_wait_settlement_pending" ? sourceFailureCode! : exhausted ? ("native_session_retry_exhausted" as const) @@ -8040,6 +8041,7 @@ export function nativeSessionFailureSourceCode( ): | "native_permission_declined" | "native_provider_terminal_failed" + | "native_governed_wait_settlement_pending" | typeof NATIVE_PROVIDER_STATE_INCOMPATIBLE | typeof NATIVE_PROVIDER_SIDE_EFFECT_PENDING | "native_provider_approval_required" @@ -8072,6 +8074,7 @@ export function nativeSessionFailureSourceCode( | "local_runtime_api_listener_unavailable" | "native_session_interrupted" { if (error instanceof NativePermissionDeclinedError) return "native_permission_declined"; + if (error instanceof NativeGovernedWaitSettlementPendingError) return "native_governed_wait_settlement_pending"; if (error instanceof NativeProviderStateIncompatibleError) return NATIVE_PROVIDER_STATE_INCOMPATIBLE; if (error instanceof NativeProviderSideEffectPendingError) return NATIVE_PROVIDER_SIDE_EFFECT_PENDING; if (error instanceof NativeProviderTerminalFailure) { @@ -11897,6 +11900,8 @@ async function executePaperclipNativeSessionAttempt( ? "The computer provider rejected access. Restore the environment's credentials or permissions before retrying. Automatic retries are stopped." : sourceFailureCode === "native_provider_terminal_failed" ? "The provider session is permanently unusable. Verify stopped execution, completed actions, and task context before starting a linked continuation." + : sourceFailureCode === "native_governed_wait_settlement_pending" + ? "The work and waiting state are saved. Confirm shutdown of the retained execution and settle that same handoff without repeating provider work." : sourceFailureCode === "native_provider_model_rejected" && !recoveryEvidence.providerSessionEstablished ? NATIVE_STARTUP_MODEL_REJECTED_ACTION : sourceFailureCode === NATIVE_PROVIDER_STATE_INCOMPATIBLE diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts index 9c6506cac88..163de754320 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts @@ -327,7 +327,7 @@ describe("PaperclipControlPlanePort conformance", () => { await rm(outputStorageRoot, { recursive: true, force: true }); }); - it.each(["settles", "checkpoint timeout", "unscoped terminal"] as const)("retains a durably recorded governed terminal while its postcommit observer is pending: %s", async mode => { + it.each(["settles", "checkpoint timeout", "unscoped terminal", "verified shutdown"] as const)("settles a durable governed wait through its recorded terminal or shutdown: %s", async mode => { const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, issueId: randomUUID(), runId: randomUUID(), sessionId: randomUUID() }; const runnerId = randomUUID(); const fixtureContractId = randomUUID(); @@ -383,9 +383,11 @@ describe("PaperclipControlPlanePort conformance", () => { async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async startTurn() { starts++; return { turnId: "park-turn" }; }, cancel() { return { cleanup: Promise.resolve() }; }, - async *events() { yield event(1, "item.completed", { kind: "dynamicToolCall" }); terminalEmitted = true; yield event(2, "turn.interrupted", { status: "interrupted", error: null }); }, + async settleCancelledTurn() { return { runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: "park-turn" }; }, + async *events() { yield event(1, "item.completed", { kind: "dynamicToolCall" }); terminalEmitted = true; + if (mode !== "verified shutdown") yield event(2, "turn.interrupted", { status: "interrupted", error: null }); }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, - activeTurnId: terminalEmitted ? null : "park-turn", cursor: terminalEmitted ? "2" : "1" }; }, + activeTurnId: terminalEmitted ? null : "park-turn", cursor: terminalEmitted && mode !== "verified shutdown" ? "2" : "1" }; }, async result() { return null; }, async close() {}, }; }, }; @@ -409,6 +411,19 @@ describe("PaperclipControlPlanePort conformance", () => { const execution = executeNativeSession(options); const outcome = execution.then(value => ({ value }), error => ({ error })); try { + if (mode === "verified shutdown") { + expect(await outcome).toMatchObject({ value: { result: yielded, terminal: { runTerminalState: "succeeded" } } }); + expect(starts).toBe(1); + expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, identity.runId))).toHaveLength(1); + expect((await port.loadSessionCheckpoint())?.governedWait?.settlement).toEqual({ + runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: "park-turn", + }); + expect((await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, identity.runId))) + .some(row => row.eventType.startsWith("turn."))).toBe(false); + expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, interaction.id)))[0]) + .toMatchObject({ status: "pending", result: null, effectiveResolverPolicy: "human_only", sourceRunId: identity.runId }); + return; + } await Promise.race([recorded, outcome.then(result => { if ("error" in result) throw result.error; throw new Error("run settled before recording terminal"); })]); const rows = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, identity.runId)); expect(rows.some(row => row.eventType === "turn.interrupted")).toBe(true); diff --git a/server/src/services/task-interruption-presentation.test.ts b/server/src/services/task-interruption-presentation.test.ts index e3e20509cc0..f086b7cb4a2 100644 --- a/server/src/services/task-interruption-presentation.test.ts +++ b/server/src/services/task-interruption-presentation.test.ts @@ -7,6 +7,25 @@ const run = { id: "run", agentId: "agent", status: "failed", scheduledRetryAt: n resultJson: { projectAccessDenied: { projectId: "22222222-2222-4222-8222-222222222222" } } }; const agent = { status: "idle", pauseReason: null }; +describe("saved governed wait shutdown", () => { + const now = Date.parse("2026-11-09T12:00:00Z"); + const pending = { ...run, runtimeMode: "native", errorCode: "native_governed_wait_settlement_pending", resultJson: null }; + it("names automatic shutdown settlement without offering a duplicate Retry", () => { + expect(projectTaskInterruption(issue, { run: { ...pending, nativeResumeAt: new Date(now + 30_000) }, agent, + paused: false, approval: false }, now)).toMatchObject({ state: "working", canRetry: false, + retryWait: { reason: "governed_wait_settlement", message: "Your work is saved. Paperclip is finishing shutdown." } }); + }); + it("explains exhausted shutdown recovery while preserving the saved work", () => { + expect(projectTaskInterruption(issue, { run: pending, agent, paused: false, approval: false }, now)) + .toMatchObject({ state: "failed", canRetry: false, retryWait: { reason: "governed_wait_settlement", + message: "Paperclip couldn’t confirm shutdown. Your work is saved; continuation is paused." } }); + }); + it("keeps genuine approvals authoritative", () => { + expect(projectTaskInterruption(issue, { run: pending, agent, paused: false, approval: true }, now)) + .toMatchObject({ state: "waiting_for_user", userAction: "approval" }); + }); +}); + describe("queued dependency wait presentation", () => { const waitingRun = { ...run, status: "cancelled", runtimeMode: "legacy" as const, startedAt: null, errorCode: "issue_dependencies_blocked", resultJson: { diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 4c77ab82c09..9f6c9130273 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -261,6 +261,13 @@ export function projectTaskInterruption(issue: ProjectionIssue, input: Projectio // dispatch runs. A resume that never ran falls through to one Retry. const nativeResumeAt = run.nativeResumeAt?.getTime(); const retryAt = system?.nextAttemptAt ?? nativeResumeAt ?? run.scheduledRetryAt?.getTime(); + if (run.runtimeMode === "native" && run.errorCode === "native_governed_wait_settlement_pending") { + const settling = recoveryRunning || Boolean(retryAt && now - retryAt < NATIVE_RESUME_DISPATCH_GRACE_MS); + return { ...base, state: settling ? "working" : "failed", canRetry: false, + retryWait: { reason: "governed_wait_settlement", message: settling + ? "Your work is saved. Paperclip is finishing shutdown." + : "Paperclip couldn’t confirm shutdown. Your work is saved; continuation is paused." } }; + } if (retryAt && retryAt > now) return { ...base, state: "retry_scheduled", nextRetryAt: new Date(retryAt).toISOString(), canRetry: !preparationPending && !nativeResumeAt, ...(system?.nextAttemptAt && recoveryStep ? { recoveryStep } : {}) }; diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 732c76e741d..76be2f77175 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -189,6 +189,7 @@ export const validatePrpStructuredRunResult = runner.validatePrpStructuredRunResult; export const NativeProviderTerminalFailure = runner.NativeProviderTerminalFailure; +export const NativeGovernedWaitSettlementPendingError = runner.NativeGovernedWaitSettlementPendingError; export const NativeProviderStateIncompatibleError = runner.NativeProviderStateIncompatibleError; export const NativeProviderSideEffectPendingError = runner.NativeProviderSideEffectPendingError; export const NativeRunnerSourceReconciliationRequiredError = runner.NativeRunnerSourceReconciliationRequiredError; diff --git a/ui/src/pages/Inbox.test.tsx b/ui/src/pages/Inbox.test.tsx index be8c6382997..fc6e2c68991 100644 --- a/ui/src/pages/Inbox.test.tsx +++ b/ui/src/pages/Inbox.test.tsx @@ -2092,6 +2092,22 @@ describe("FailedRunInboxRow", () => { container.remove(); }); + it.each(["working", "failed"] as const)("shows saved-work shutdown guidance while %s without Retry", state => { + const root = createRoot(container); + const message = state === "working" ? "Your work is saved. Paperclip is finishing shutdown." + : "Paperclip couldn’t confirm shutdown. Your work is saved; continuation is paused."; + const issue = createIssue({ interruption: { intentId: "saved-input", responsibility: "system", state, + nextRetryAt: null, canRetry: false, userAction: null, sourceRunIds: ["run-1"], observedRunId: "run-1", + retryWait: { reason: "governed_wait_settlement", message } } }); + act(() => root.render( {}} onRetry={() => {}} isRetrying={false} />)); + expect(container.textContent).toContain(message); + expect(container.textContent).not.toContain("Couldn’t finish this attempt"); + expect(container.textContent).not.toContain("Retry"); + act(() => root.unmount()); + }); + it("uses task presentation instead of infrastructure errors and keeps Retry in the task", () => { const root = createRoot(container); const issue = createIssue({ interruption: { intentId: "saved-input", responsibility: "system", state: "failed", diff --git a/ui/src/pages/Inbox.tsx b/ui/src/pages/Inbox.tsx index 47d4cb4104b..4ea945e8e52 100644 --- a/ui/src/pages/Inbox.tsx +++ b/ui/src/pages/Inbox.tsx @@ -397,7 +397,9 @@ export function FailedRunInboxRow({ const interruption = issue?.interruption; useSecondTick(interruption?.state === "retry_scheduled"); const retrySeconds = Math.max(0, Math.ceil((Date.parse(interruption?.nextRetryAt ?? "") - Date.now()) / 1000)); - const displayError = isRetrying || interruption?.state === "working" ? "Thinking" + const displayError = isRetrying ? "Thinking" + : interruption?.retryWait ? interruption.retryWait.message + : interruption?.state === "working" ? "Thinking" : interruption?.state === "retry_scheduled" ? `Trying again in ${retrySeconds}s…` : interruption?.state === "failed" ? "Couldn’t finish this attempt. Your message is saved." : interruption?.state === "waiting_for_user" ? ({