From 6cfc79b50dc89207240328e6ed3f86c29a860f6d Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 2 Oct 2026 17:27:37 -0500 Subject: [PATCH] Require complete Pi control-plane steering settlement Co-Authored-By: Paperclip --- .../2026-10-02-pi-production-readiness.md | 33 ++++++++++++++++++- tests/runner-e2e/catalog.ts | 4 +-- tests/runner-e2e/pi-controls-evidence.test.ts | 21 ++++++------ tests/runner-e2e/pi-controls-evidence.ts | 7 +++- tests/runner-e2e/pi-controls-flow.test.ts | 2 +- tests/runner-e2e/pi-controls-test-fixture.ts | 8 ++++- 6 files changed, 59 insertions(+), 16 deletions(-) diff --git a/doc/plans/2026-10-02-pi-production-readiness.md b/doc/plans/2026-10-02-pi-production-readiness.md index b8a064549a6..75614c81fc1 100644 --- a/doc/plans/2026-10-02-pi-production-readiness.md +++ b/doc/plans/2026-10-02-pi-production-readiness.md @@ -22,7 +22,7 @@ qualification. The existing draft stack must be reviewed in dependency order. | Gate | Acceptance evidence | Current result | | --- | --- | --- | | Runtime identity and admission | Exact runtime/profile/model; verified effective thinking; startup below the 60-second admission limit; unsupported configuration fails before a prompt | Deterministic checks and ARM installed startup pass. Recheck final source on all supported hosts. | -| Live lifecycle | Pending native question survives controller restart in the original process; one creation and resolution; stale answer after provider death rejected; Stop retires owned processes; three warm turns keep process identity | Installed local restart and three-turn warm continuity pass at `f5f57e380`. Pending-permission Stop and stale response rejection pass at `03dd6ef93`. Provider-death evidence remains required. | +| Live lifecycle | Pending native question survives controller restart in the original process; one creation and resolution; stale answer after provider death rejected; Stop retires owned processes; three warm turns keep process identity | Installed profile-14 same-turn steering passes at `dc2b053f3`. Restart and three-turn warm continuity pass at historical `f5f57e380`; pending-permission Stop and stale response rejection pass at `03dd6ef93`. Final-source lifecycle and provider-death evidence remain required. | | Product workflows | All 26 explicit Pi Product E2E cells (13 local, 13 Daytona); screenshots, public state, independent artifacts, terminal and cleanup evidence | Historical partial passes exist. Full final-source campaign remains required. | | Runner protocol | Pi roster passes through the native packaged runner and authenticated mock control plane; lifecycle/denial/control cases remain distinct from Product tests | Installed get-task-context passes at `f5f57e380`; context-before-action times out. Five cells remain unexecuted. | | Installed distribution | Public CLI/server tars on ARM Mac, Intel Mac and Linux; normal Pi setup; exact Linux companion and immutable Daytona image imported without binary override | Historical ARM and Intel public installations pass. Linux CI retains `session_handshake_timeout`; queued fleet jobs were cancelled. Final-source platform receipts, Linux companion and Daytona image remain required. | @@ -45,6 +45,21 @@ prices are estimates. Unpriced usage must stay unpriced in the ledger and UI. ## Evidence so far +- At `dc2b053f3`, the fresh installed profile-14 steering journey receives a + canonical pass in 52 seconds, including cleanup. It proves the exact browser + comment, one acknowledgement, denial of the original native write, the hidden + instruction in the persisted final comment, succeeded/Done, process retirement + and continuous no-effect evidence through cleanup. The original failed grades + stay unchanged. No automatic retry or fallback key occurs. +- Final controls definitions advance to version 6. Steering now requires both + linked control-plane records, the matching accepted result, and succeeded/ + completed/done terminal values. Stop keeps its separate cancellation contract. + Missing, foreign, duplicate, premature and contradictory records fail. All + 129 controls/flow/catalog tests and Product E2E typecheck pass. The successful + installed receipt also passes this stricter replay, with no provider call. + The version-5 canonical pass remains version-5 evidence; the full final-source + matrix remains a release gate. + - At `603ad3726`, the installed steering journey posts Steer (HTTP 200) and Deny (HTTP 202) to the original request. The provider consumes the hidden instruction, produces its exact final marker, and reaches succeeded/Done. @@ -187,6 +202,22 @@ prices are estimates. Unpriced usage must stay unpriced in the ledger and UI. the verifier gives that launch's private runtime snapshots the same bounded scratch capacity. This is not counted as a passing Linux receipt yet. +## Remaining work in order + +1. Diagnose Linux closed admission from its real startup and cleanup evidence. + Resolve `session_handshake_timeout` and classify the observed Mac ownership + contention. Produce the normal Linux public-install and companion receipts, + then import an immutable exact-source Daytona image without an override. + Do not accept a timeout as an installation pass or widen the admission limit. +2. Resolve Runner `context-before-action` with a concrete behavioral correction. + It currently repeats context calls without the required progress mutation. + Do not run an unchanged paid retry. Then execute the remaining explicit + Runner cases and the full 26-cell Product matrix on the frozen candidate. +3. Close the prerequisite review stack and final CI gates. Retain the local + Codex interrupt-recovery failure until it is resolved or its host-specific + cause is established. Publish only the qualified artifacts, then use the + bounded operator rollout below. No merge or release is authorized here. + ## Rollout and rollback Release only the exact qualified package set. Begin with a small operator-owned diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index a2857461162..27e07a246c2 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1090,9 +1090,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Pending native-write Stop and browser-originated same-turn steering, with exact control receipts and independent retirement/no-effect evidence.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "pi"), environments: runnerEnvironments, tasks: piControlTasks, expectedMatrixSize: 4, - definitionMetadata: { version: 5, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, + definitionMetadata: { version: 6, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, nativeArguments: "streamed-until-exact-target", - controlPlaneSettlement: "scoped-result-and-terminal-after-runner", + controlPlaneSettlement: "required-scoped-result-and-terminal-after-runner", steeringComment: "exact-browser-submitted-markdown", steeringDispatch: "require-public-api-acceptance", pending: "paperclip.e2e.pi-control-pending.v1", stop: "paperclip.e2e.pi-stop-settlement.v1", steering: "paperclip.e2e.pi-steering-settlement.v1", diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index 44120701ddb..22d0a75b2f0 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -112,14 +112,7 @@ function steered() { } describe("Pi same-turn steering", () => { function withControlSettlement() { - const s = steered(); - s.f.append("run.result.accepted", { result: { schema: "paperclip.run_result.v1" } }, { - sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 1, sourceEventId: "source:control:run:1", - }); - s.f.append("run.terminal", { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded" }, { - sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 2, sourceEventId: "source:control:run:2", - }); - return s; + return steered(); } it("keeps exact runner proof when the control plane records result and terminal events", () => { expect(withControlSettlement().read()).toMatchObject({ nativeFollowUpTested: false }); @@ -132,6 +125,14 @@ describe("Pi same-turn steering", () => { ["reordered control sequence", s => { event(s.f.events.at(-1)!).sourceSeq = 1; event(s.f.events.at(-1)!).sourceEventId = "source:control:run:1"; }], ["duplicate control terminal", s => { s.f.append("run.terminal", { schema: "paperclip.prp.terminal.v1" }, { sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 3, sourceEventId: "source:control:run:3" }); }], ["unbound control result", s => { event(s.f.events.at(-2)!).payload.result.schema = "foreign"; }], + ["both control records missing", s => { s.f.events.splice(-2); }], + ["accepted result missing", s => { s.f.events.splice(-2, 1); }], + ["control terminal missing", s => { s.f.events.pop(); }], + ["wrong control summary", s => { event(s.f.events.at(-2)!).payload.result.summary = "Finished"; }], + ["unfinished control result", s => { event(s.f.events.at(-2)!).payload.result.reportedWorkDisposition = "in_progress"; }], + ["failed control terminal", s => { event(s.f.events.at(-1)!).payload.runTerminalState = "failed"; }], + ["interrupted control turn", s => { event(s.f.events.at(-1)!).payload.turnTerminalState = "interrupted"; }], + ["unfinished control terminal", s => { event(s.f.events.at(-1)!).payload.reportedWorkDisposition = "in_progress"; }], ["control terminal before runner completion", s => { for (const row of s.f.events.slice(3, 6)) row.seq += 2; s.f.events.at(-2)!.seq = 4; s.f.events.at(-1)!.seq = 5; @@ -189,8 +190,8 @@ describe("Pi controls catalog admission", () => { const pi = runnerMatrix.find(c => c.profile.qualificationCandidate === "pi")!.profile; expect(pi.modelQualification?.qualificationId).toBe("pi:0.0.33:1.0.0:openrouter"); expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 4, profileVersion: 14 }); - expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 5, profileVersion: 14, - controlPlaneSettlement: "scoped-result-and-terminal-after-runner" }); + expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 6, profileVersion: 14, + controlPlaneSettlement: "required-scoped-result-and-terminal-after-runner" }); expect(runnerSuites.find(s => s.id === "extended-harnesses")!.definitionMetadata).toMatchObject({ version: 2 }); for (const cell of runnerMatrix.filter(cell => cell.profile.qualificationCandidate === "pi")) { const agent = cell.profile.buildAgent({ environmentId: "environment", environmentFixtureId: cell.environment.id, workspacePath: "/workspace", executionId: cell.id, secretRefs: { OPENROUTER_API_KEY: { type: "secret_ref", secretId: "synthetic", version: "latest" } } }); diff --git a/tests/runner-e2e/pi-controls-evidence.ts b/tests/runner-e2e/pi-controls-evidence.ts index e1b84b14d98..af6c3a2cef2 100644 --- a/tests/runner-e2e/pi-controls-evidence.ts +++ b/tests/runner-e2e/pi-controls-evidence.ts @@ -77,7 +77,7 @@ function origin(events: readonly unknown[], scope: PiControlScope) { // Read-only orientation/bootstrap can precede the write. Never exempt another // edit, shell execution, or unknown native operation as an alleged bootstrap. requireProof(native.every(x => rec(x.event.payload).executionId === executionId || (x.event.payload.operation === "read" && x.row.seq < Math.min(card.row.seq, started[0]!.row.seq))), "extra native operation"); - return { rows, card, request, started: started[0]!, executionId, stream }; + return { rows, control, card, request, started: started[0]!, executionId, stream }; } export function observePiControlPending(input: PiControlState & { scope: PiControlScope }): PiControlPending { const { run, issue, scope } = input, p = origin(input.events, scope); @@ -138,6 +138,11 @@ export function readPiSteeringAcknowledgement(input: PiControlState & { pending: } export function readPiSteeringSettlement(input: PiControlState & { pending: PiControlPending; commentId: string; queueId: string; marker: string; finalMessage: string }) { const ack = readPiSteeringAcknowledgement(input), p = retained(input), b = input.pending; + const accepted = rec(rec(p.control[0]?.event.payload).result), terminalResult = rec(p.control[1]?.event.payload); + requireProof(p.control.length === 2 && p.control[0]!.event.eventType === "run.result.accepted" && p.control[1]!.event.eventType === "run.terminal" + && accepted.reportedWorkDisposition === "done" && accepted.summary === input.marker + && terminalResult.runTerminalState === "succeeded" && terminalResult.turnTerminalState === "completed" && terminalResult.reportedWorkDisposition === "done", + "complete matching control-plane settlement required"); const closed = p.rows.filter(x => closureTypes.has(x.event.eventType)), terminal = p.rows.filter(x => terminalTypes.has(x.event.eventType)); const failed = p.rows.filter(x => x.event.eventType === "tool.execution.completed" && x.event.payload.executionId === b.executionId); requireProof(closed.length === 1 && closed[0]!.event.eventType === "runtime_request.resolved" && closed[0]!.event.payload.requestId === b.requestId && closed[0]!.event.payload.turnId === b.turnId diff --git a/tests/runner-e2e/pi-controls-flow.test.ts b/tests/runner-e2e/pi-controls-flow.test.ts index dd2a0ff8fd7..469f6c25d78 100644 --- a/tests/runner-e2e/pi-controls-flow.test.ts +++ b/tests/runner-e2e/pi-controls-flow.test.ts @@ -97,7 +97,7 @@ async function exercise(taskId: string, remote: boolean, failure?: "mutation" | if (failure === "missing-ack") f.run.resultJson.queuedSteeringAcknowledgements = {}; postBrowser("/api/issues/issue/queued-comments/comment/steer", { queueId: "queue", targetRunId: "run", revision: "revision" }); } else if (kind === "deny") { - browserDeclines++; expect(browserSteers).toBe(1); expect(stopCase).toBe(false); f.finish(); + browserDeclines++; expect(browserSteers).toBe(1); expect(stopCase).toBe(false); f.finish(steeringMarker); f.run.resultJson.presentationDecision = { commentId: "final", reason: "fixture-public-presentation" }; postBrowser("/api/heartbeat-runs/run/runtime-requests/request/resolve", { turnId: "turn", requestKind: "permission_approval", resolution: { action: "decline" } }); } else throw new Error(`Unexpected click ${kind}`); diff --git a/tests/runner-e2e/pi-controls-test-fixture.ts b/tests/runner-e2e/pi-controls-test-fixture.ts index 1f37d498751..f121f06af4b 100644 --- a/tests/runner-e2e/pi-controls-test-fixture.ts +++ b/tests/runner-e2e/pi-controls-test-fixture.ts @@ -48,11 +48,17 @@ export function piControlFixture(target = "target.txt") { run.resultJson.queuedSteeringAcknowledgements = { [commentId]: { status: "acknowledged", queueId, turnId: "turn", acknowledgedAt: "2026-10-01T00:00:01Z" } }; append("item.completed", { kind: "steering_acknowledgement", status: "acknowledged", mode: "steer", text: "Steering acknowledged for the active turn." }, { itemId: `turn:steer:${commentId}` }); } - function finish() { + function finish(finalMarker = marker) { append("runtime_request.resolved", { requestId: "request", turnId: "turn", action: "decline" }); append("tool.execution.completed", { ...tool, status: "failed", output: "Pi operation was denied or cancelled" }); append("turn.completed", { status: "completed", error: null }); run.status = "succeeded"; issue.status = "done"; + append("run.result.accepted", { result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: finalMarker } }, { + sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 1, sourceEventId: "source:control:run:1", + }); + append("run.terminal", { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: "done" }, { + sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 2, sourceEventId: "source:control:run:2", + }); } return { scope, events, run, issue, state, pending, row, append, request, tool, cancel, responses, steer, finish, marker, commentId, queueId }; }