diff --git a/doc/plans/2026-10-02-pi-production-readiness.md b/doc/plans/2026-10-02-pi-production-readiness.md index 67cb65a30a..266665746d 100644 --- a/doc/plans/2026-10-02-pi-production-readiness.md +++ b/doc/plans/2026-10-02-pi-production-readiness.md @@ -5,6 +5,37 @@ set of release gates. Pi remains a candidate until every required gate passes. Do not expand this work to additional models, widgets, images, Cursor or Copilot qualification. The existing draft stack must be reviewed in dependency order. +## Case corrections — 2026-10-05 + +The latest completed frozen-runtime proofs are **28/33: Product 21/26 and +Runner 7/7**. These counts supersede older snapshots below. The two corrected +Runner prompts pass at private definitions revision `7b112ffe3` with unchanged +graders, one attempt per correction and zero automatic retries. Their original +failures remain retained. The Mac file-edit correction also passes. + +The provider-death fixture now admits the production runner's stable symlink +while still checking the resolved bytes, live Node inode and exact Pi-child +pidfd. All 21 Linux fault tests and 297 Linux fixture checks pass. The next live +attempt signals the correct Pi child and passes cleanup, but exposes a runtime +bug: its provider-loss expiry creates a durable question fallback that the +server mistakes for successful yielded completion. + +The runtime correction preserves that fallback while preventing governed-wait +settlement after provider loss. The Runner emits `turn.failed` for the lost +provider instead of `turn.interrupted`. The regression fails before the fix; +169 Runner tests and all 574 native-server tests pass afterward. This changes +shipping runtime inputs. Existing `0bd040093` artifacts and paid passes do not +qualify the corrected runtime; fresh public artifacts and source-bound live +qualification remain required. + +The memory prompt now states its nonce-plus-final-LF contract in plain text and +bounds the native write/read sequence. A free call to the frozen Pi tools +retains all 33 bytes, including the LF. The corrected local live attempt still +times out after native policy rejects paths outside the assigned roots. +Cleanup passes. Both failed corrected local attempts remain failed; the memory +case is not qualified. No byte assertion, timeout, model or permission boundary +is relaxed. Production remains held; no merge or release is authorized. + ## Frozen target - Pi: `@earendil-works/pi-coding-agent@1.0.0`. diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts index 470bd66d71..a4095023ab 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -851,7 +851,7 @@ describe("HarnessDriverBackend", () => { expect(events.map((event) => event.eventType)).toEqual([ "runtime_request.created", "runtime_request.expired", - "turn.interrupted", + "turn.failed", ]); } await session.close({ reason: "fixture complete" }); @@ -898,7 +898,7 @@ describe("HarnessDriverBackend", () => { await session.close({ reason: "fixture complete" }); }); - it("emits one non-replayable input expiration and terminal wait after provider loss", async () => { + it("expires pending input without converting provider loss into a successful wait", async () => { const questionSet = { schema: "paperclip.question_set.v1" as const, questions: [{ id: "target", prompt: "Which target?", required: true, answerMode: "text" as const }], @@ -940,11 +940,14 @@ describe("HarnessDriverBackend", () => { }, } }); await expect(iterator.next()).resolves.toMatchObject({ value: { - eventType: "turn.interrupted", + eventType: "turn.failed", sourceSeq: 3, payload: { reason: "provider_process_lost" }, } }); await expect(iterator.next()).resolves.toMatchObject({ done: true }); + await expect(session.snapshot()).resolves.toMatchObject({ terminal: { + turnTerminalState: "failed", runTerminalState: "failed", + } }); }); it("does not synthesize a fallback after the input was already resolved", async () => { diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index 3dd6a5b465..8babd3182a 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -507,7 +507,7 @@ class HarnessNativeSession implements NativeSession { let sourceInstanceId: string | null = null; let lastSourceSequence = 0; let sawTerminal = false; - let synthesizedDurableWait = false; + let synthesizedProviderFailure = false; let streamFailure: unknown = null; const observedPendingInputs = new Map>(); try { @@ -627,7 +627,7 @@ class HarnessNativeSession implements NativeSession { this.#rethrowProtocolIntegrity(error); return null; }); - let governedWaitTurnId: string | undefined; + let providerLossTurnId: string | undefined; for (const request of observedPendingInputs.values()) { const sourceSeq = Math.max(lastSourceSequence, snapshot?.lastSourceSequence ?? 0) + 1; @@ -635,7 +635,7 @@ class HarnessNativeSession implements NativeSession { const requestId = String(request.requestId); const turnId = typeof request.turnId === "string" ? request.turnId : undefined; - governedWaitTurnId ??= turnId; + providerLossTurnId ??= turnId; const itemId = typeof request.itemId === "string" ? request.itemId : requestId; yield { @@ -667,16 +667,18 @@ class HarnessNativeSession implements NativeSession { }, }; } - if (governedWaitTurnId) { + if (providerLossTurnId) { + // Expiring input preserves its durable human fallback. The unexpected + // transport loss remains a failed provider execution, never a yield. const sourceSeq = Math.max(lastSourceSequence, snapshot?.lastSourceSequence ?? 0) + 1; lastSourceSequence = sourceSeq; - synthesizedDurableWait = true; + synthesizedProviderFailure = true; sawTerminal = true; this.#terminal = { schema: "paperclip.prp.terminal.v1", - turnTerminalState: "interrupted", - runTerminalState: "cancelled", + turnTerminalState: "failed", + runTerminalState: "failed", reportedWorkDisposition: "yielded", }; yield { @@ -687,16 +689,16 @@ class HarnessNativeSession implements NativeSession { sourceKind: "runner", runId: this.#input.identity.runId, normalizedSessionId: this.#input.identity.sessionId, - turnId: governedWaitTurnId, - eventType: "turn.interrupted", + turnId: providerLossTurnId, + eventType: "turn.failed", schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(), - payload: { status: "interrupted", reason: "provider_process_lost" }, + payload: { status: "failed", reason: "provider_process_lost" }, }; } } - if (streamFailure && !synthesizedDurableWait) throw streamFailure; + if (streamFailure && !synthesizedProviderFailure) throw streamFailure; } async startTurn(input: Parameters[0]) { diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index df70c10baa..00a6905e16 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -4813,6 +4813,16 @@ describe("native governed waits", () => { payload: { kind: "dynamicToolCall" }, }; + // The durable fallback preserves the question after provider loss, but + // cannot turn that lost execution into a successful governed wait. + const providerLost = { ...replayedEvent, eventType: "runtime_request.expired" as const, + payload: { reason: "provider_process_lost", replayAllowed: false } }; + const lostObservation = createGovernedWaitEventObservation(async () => waitResult); + await lostObservation.observe(providerLost, true); + expect(lostObservation.consume(providerLost)).toBeNull(); + await lostObservation.observe(replayedEvent, true); + expect(lostObservation.consume(replayedEvent)).toBeNull(); + // The event consumer can lag the provider: a commentary event emitted // before the tool began may be processed after its approval exists in DB. // It is not proof that the approval-creating tool response has settled. diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 42241b6c34..58d51f68a3 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1213,6 +1213,7 @@ export function createGovernedWaitEventObservation( resolvePending: () => Promise, ) { const pendingTools = new Set(); + let providerLost = false; let generation = 0; let observation: { sourceInstanceId: string; @@ -1226,6 +1227,7 @@ export function createGovernedWaitEventObservation( const currentGeneration = ++generation; observation = null; const payload = record(event.payload); + providerLost ||= event.eventType === "runtime_request.expired" && payload.reason === "provider_process_lost"; const kind = payload.kind; const tool = ["dynamicToolCall", "mcpToolCall", "commandExecution"].includes(String(kind)); if (event.itemId) { @@ -1241,7 +1243,9 @@ export function createGovernedWaitEventObservation( if (event.eventType === "item.completed" && ( pendingTools.size > 0 || (!tool && !(kind === "agentMessage" && payload.channel === "final")) )) return; - if (!eligible) return; + // A replacement question preserves human input after a provider crash; + // it does not authorize a successful suspension of the failed execution. + if (providerLost || !eligible) return; const result = await resolvePending(); if (generation !== currentGeneration || result === null) return; // If the interaction is answered after this read, parking remains the