diff --git a/doc/plans/2026-10-02-pi-production-readiness.md b/doc/plans/2026-10-02-pi-production-readiness.md index f2aed4ae5fe..b8a064549a6 100644 --- a/doc/plans/2026-10-02-pi-production-readiness.md +++ b/doc/plans/2026-10-02-pi-production-readiness.md @@ -45,24 +45,43 @@ prices are estimates. Unpriced usage must stay unpriced in the ledger and UI. ## Evidence so far -- At `342287678`, normal public profile-14 admission passes on ARM Mac in - 23.8 seconds and Intel Mac in 53.5 seconds. Fresh installed steering returns - HTTP 200 and retains one exact correlation-bound acknowledgement. The full - journey fails at Deny: steering splits the transcript display and the UI - incorrectly marks the older, still-live permission cancelled. A real widget - regression reproduces the missing button. The correction derives request - state from the whole run, carries pending cards to the live tail, and keeps - closed cards at their original position. All 281 focused UI/projection and - performance checks, UI typecheck and token gates pass. The paid failure and - cleanup grade remain unchanged; all seven journalled owned processes later - retire. A new installed journey must verify actual denial and consumption. -- Full workspace build and typecheck pass at `342287678`; core native tests - pass 333 cases. The broader native integration run fails one Codex interrupt - recovery case with `provider startup ownership remains unadmitted`; isolated - diagnosis reproduces it. Do not count that broader run as passing. Current - CI passes its other checks and Greptile is 5/5, but Linux Canary retains - `session_handshake_timeout`. The exact-source local image build cannot reach - the Docker daemon and produces no usable image. Linux and Daytona remain held. +- At `603ad3726`, the installed steering journey posts Steer (HTTP 200) and + Deny (HTTP 202) to the original request. The provider consumes the hidden + instruction, produces its exact final marker, and reaches succeeded/Done. + Its canonical grade remains failed: the oracle rejects the legitimate + `run.result.accepted` and `run.terminal` control-plane records as non-runner + events. All seven journalled provider processes retire with no target effect. + The corrected oracle validates those two records against the same run, turn, + session and linked control producer, after the one native terminal. It still + rejects foreign, duplicate, reordered and premature records. All 121 controls, + flow and catalog tests pass, as does Product E2E typecheck. Replaying the exact + retained public evidence passes the corrected oracle; replay does not regrade + the original attempt or prove a fresh cleanup receipt. Controls definitions + advance to version 5. A fresh installed journey remains the qualification gate. +- At `603ad3726`, public profile-14 setup passes on ARM and Intel Mac. The + first ARM closed-admission probe rejects with `provider_lifetime_owned`; + a separate credential-free diagnostic with retained state passes in 35.0 + seconds. Preserve both observations; ownership contention remains unclassified. + Intel closed admission passes in 54.1 seconds. Neither host submits a prompt. +- Full workspace build passes at `3728bb45f`; only the tested UI request-order + correction changes executable code afterward. Workspace typecheck and UI + build pass at `603ad3726`. All 282 UI/projection/performance tests and token + gates pass. Core native tests pass 333 cases. The broader local native + integration run fails one Codex interrupt recovery case with `provider startup + ownership remains unadmitted`; isolated diagnosis reproduces it. Do not count + that broader run as passing. +- At `603ad3726`, 52 CI checks pass and Greptile is 5/5 with both findings + resolved. Linux Canary is the one failed check: `session_handshake_timeout`. + The local exact-source Linux build compiles but exceeds its fixed 30-minute + deadline during image import. It produces no verified usable image. Its + task-owned builder is stopped, its cache and failed receipt remain, and no + provider credentials or calls occur. Linux and Daytona remain held. +- At `342287678`, installed steering proves delivery and one acknowledgement, + but the UI marks the still-pending permission cancelled and hides Deny. The + correction uses whole-run lifecycle state, carries pending cards to the live + tail, and leaves closed cards at their original position. Older carried cards + precede newer requests. The real widget regression clicks Deny for the + original run, request and provider turn. The paid failure stays failed. - At `36e712104`, installed steering returns HTTP 200 and the native command journal records accepted delivery. The public API retains the correct correlation-bound acknowledgement at source sequence 65 and also a diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 4b66ef39ada..a2857461162 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1090,8 +1090,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Pending native-write Stop and browser-originated same-turn steering, with exact control receipts and independent retirement/no-effect evidence.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "pi"), environments: runnerEnvironments, tasks: piControlTasks, expectedMatrixSize: 4, - definitionMetadata: { version: 4, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, + definitionMetadata: { version: 5, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, nativeArguments: "streamed-until-exact-target", + controlPlaneSettlement: "scoped-result-and-terminal-after-runner", steeringComment: "exact-browser-submitted-markdown", steeringDispatch: "require-public-api-acceptance", pending: "paperclip.e2e.pi-control-pending.v1", stop: "paperclip.e2e.pi-stop-settlement.v1", steering: "paperclip.e2e.pi-steering-settlement.v1", diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index 1cd582f9d1c..44120701ddb 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -111,6 +111,34 @@ function steered() { return { f, binding, ack, read: () => readPiSteeringSettlement({ ...f.state(), ...binding }) }; } describe("Pi same-turn steering", () => { + function withControlSettlement() { + const s = steered(); + s.f.append("run.result.accepted", { result: { schema: "paperclip.run_result.v1" } }, { + sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 1, sourceEventId: "source:control:run:1", + }); + s.f.append("run.terminal", { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded" }, { + sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 2, sourceEventId: "source:control:run:2", + }); + return s; + } + it("keeps exact runner proof when the control plane records result and terminal events", () => { + expect(withControlSettlement().read()).toMatchObject({ nativeFollowUpTested: false }); + }); + it.each([ + ["foreign control producer", (s: ReturnType) => { event(s.f.events.at(-1)!).sourceInstanceId = "foreign:control"; event(s.f.events.at(-1)!).sourceEventId = "foreign:control:run:2"; }], + ["foreign control turn", s => { event(s.f.events.at(-1)!).turnId = "foreign"; }], + ["foreign control session", s => { event(s.f.events.at(-1)!).normalizedSessionId = "foreign"; }], + ["unknown control event", s => { s.f.events.at(-1)!.eventType = event(s.f.events.at(-1)!).eventType = "runtime_request.resolved"; }], + ["reordered control sequence", s => { event(s.f.events.at(-1)!).sourceSeq = 1; event(s.f.events.at(-1)!).sourceEventId = "source:control:run:1"; }], + ["duplicate control terminal", s => { s.f.append("run.terminal", { schema: "paperclip.prp.terminal.v1" }, { sourceKind: "control_plane", sourceInstanceId: "source:control", sourceSeq: 3, sourceEventId: "source:control:run:3" }); }], + ["unbound control result", s => { event(s.f.events.at(-2)!).payload.result.schema = "foreign"; }], + ["control terminal before runner completion", s => { + for (const row of s.f.events.slice(3, 6)) row.seq += 2; + s.f.events.at(-2)!.seq = 4; s.f.events.at(-1)!.seq = 5; + }], + ] satisfies Array<[string, (s: ReturnType) => void]>)("rejects %s", (_name, mutate) => { + const s = withControlSettlement(); mutate(s); expect(s.read).toThrow(); + }); it("calibrates against the actual Product ACP facade producer, not Rust's raw transport echo", async () => { const f = piControlFixture(), pending = f.pending(), calls: Row[] = []; // Only the transport and event sink are doubles. Run the actual public @@ -156,11 +184,13 @@ describe("Pi controls catalog admission", () => { expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow(); }); it("pins the Pi 1 profile and versioned coverage while retaining active Stop identity", () => { - // Pi 1/profile 13 changes profile-bearing definitions. Coverage v4/v2 adds + // Pi 1/profile 14 changes profile-bearing definitions. Coverage v4/v2 adds // provider death, pending restart and the strict file oracle; no runtime admission is promoted. const pi = runnerMatrix.find(c => c.profile.qualificationCandidate === "pi")!.profile; expect(pi.modelQualification?.qualificationId).toBe("pi:0.0.33:1.0.0:openrouter"); - expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 4, profileVersion: 13 }); + expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 4, profileVersion: 14 }); + expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 5, profileVersion: 14, + controlPlaneSettlement: "scoped-result-and-terminal-after-runner" }); expect(runnerSuites.find(s => s.id === "extended-harnesses")!.definitionMetadata).toMatchObject({ version: 2 }); for (const cell of runnerMatrix.filter(cell => cell.profile.qualificationCandidate === "pi")) { const agent = cell.profile.buildAgent({ environmentId: "environment", environmentFixtureId: cell.environment.id, workspacePath: "/workspace", executionId: cell.id, secretRefs: { OPENROUTER_API_KEY: { type: "secret_ref", secretId: "synthetic", version: "latest" } } }); @@ -168,7 +198,7 @@ describe("Pi controls catalog admission", () => { } const hashes = Object.fromEntries(runnerSuites.filter(s => ["pi-native", "native-active-stop", "extended-harnesses", "rich-acp-warm-continuity"].includes(s.id)).map(s => [s.id, suiteDefinitionHash(s)])); expect(hashes).toEqual({ - "pi-native": "5038d59a5176ca215bc29b2d532c1d51d134442b046dd230c1e060050109964d", + "pi-native": "00f1ec9adc5980c6d147fbc8a4992ff76fb2327ad8d8dc1fc8516a69d4131bd1", "native-active-stop": "99682b2b106d816a011834fae5a944ed7729958893709d5b83a19b6f595e7e4d", "rich-acp-warm-continuity": "036c0faebc2f6eee5cd22ea38887c9c22650a83561fd08ee83473a565b11bb00", "extended-harnesses": "9814841e571cb8bb1dc5188a8577245896e0ce8c851294ac5dea9e3d42689db6", diff --git a/tests/runner-e2e/pi-controls-evidence.ts b/tests/runner-e2e/pi-controls-evidence.ts index 819e789e78c..e1b84b14d98 100644 --- a/tests/runner-e2e/pi-controls-evidence.ts +++ b/tests/runner-e2e/pi-controls-evidence.ts @@ -19,20 +19,24 @@ const hash = (v: unknown) => `sha256:${createHash("sha256").update(canonicalJson function requireProof(value: unknown, reason: string): asserts value { if (!value) throw new Error(`Pi controls: ${reason}`); } const terminalTypes = new Set(["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]); const closureTypes = new Set(["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"]); +const controlSettlementTypes = new Set(["run.result.accepted", "run.terminal"]); /** Consume the actual Product projection. Pi does not emit Cursor/Copilot native * diagnostic notices; never manufacture those to reuse another provider's oracle. */ function origin(events: readonly unknown[], scope: PiControlScope) { requireProof(Object.values(scope).every(id) && events.length > 0 && events.length <= 20_000, "bounded exact scope required"); - const rows = events.map(rec).filter(row => rec(row.payload).prpEvent !== undefined).map(row => ({ row, event: rec(rec(row.payload).prpEvent) })).sort((a, b) => a.row.seq - b.row.seq); + const projected = events.map(rec).filter(row => rec(row.payload).prpEvent !== undefined).map(row => ({ row, event: rec(rec(row.payload).prpEvent) })).sort((a, b) => a.row.seq - b.row.seq); const seqs = new Set(), sourceIds = new Set(), last = new Map(); - for (const { row, event: e } of rows) { + for (const { row, event: e } of projected) { requireProof(row.companyId === scope.companyId && row.runId === scope.runId && isValidNativePrpEnvelope(e, row.protocolSchemaVersion) - && e.sourceKind === "runner" && e.runId === scope.runId && e.eventType === row.eventType && Number.isSafeInteger(row.seq) && row.seq > 0 && !seqs.has(row.seq) + && (e.sourceKind === "runner" || (e.sourceKind === "control_plane" && controlSettlementTypes.has(e.eventType))) + && e.runId === scope.runId && e.eventType === row.eventType && Number.isSafeInteger(row.seq) && row.seq > 0 && !seqs.has(row.seq) && id(e.sourceInstanceId) && Number.isSafeInteger(e.sourceSeq) && e.sourceSeq > (last.get(e.sourceInstanceId) ?? 0) && e.sourceEventId === `${e.sourceInstanceId}:${e.runId}:${e.sourceSeq}` && !sourceIds.has(e.sourceEventId), "foreign, duplicate or reordered event"); seqs.add(row.seq); sourceIds.add(e.sourceEventId); last.set(e.sourceInstanceId, e.sourceSeq); } + const rows = projected.filter(x => x.event.sourceKind === "runner"); + const control = projected.filter(x => x.event.sourceKind === "control_plane"); const created = rows.filter(x => x.event.eventType === "runtime_request.created"); requireProof(created.length === 1, "one native permission required"); const card = created[0]!, request = rec(rec(card.event.payload).request); @@ -42,6 +46,18 @@ function origin(events: readonly unknown[], scope: PiControlScope) { && Array.isArray(request.choices) && request.choices.some((c: Row) => c.key === "decline"), "native permission identity missing"); const stream = (event: Row) => event.turnId === card.event.turnId && event.normalizedSessionId === card.event.normalizedSessionId && event.sourceInstanceId === card.event.sourceInstanceId; requireProof(rows.filter(x => x.event.turnId != null).every(x => stream(x.event)), "foreign turn, session or producer"); + // Product settlement adds a separate control-plane producer after the runner + // terminal. Validate those records without using them as native tool proof. + const terminals = rows.filter(x => terminalTypes.has(x.event.eventType)); + requireProof(control.length <= 2 && new Set(control.map(x => x.event.eventType)).size === control.length + && control.every(x => terminals.length === 1 && x.row.seq > terminals[0]!.row.seq + && x.event.sourceInstanceId === `${card.event.sourceInstanceId}:control` + && x.event.turnId === card.event.turnId && x.event.normalizedSessionId === card.event.normalizedSessionId + && (x.event.eventType === "run.result.accepted" + ? rec(rec(x.event.payload).result).schema === "paperclip.run_result.v1" + : rec(x.event.payload).schema === "paperclip.prp.terminal.v1")) + && (control.length !== 2 || (control[0]!.event.eventType === "run.result.accepted" && control[1]!.event.eventType === "run.terminal")), + "foreign, duplicate or premature control-plane settlement"); const executionId = bootstrapReadExecutionId(request.details.toolCallId); const native = rows.filter(x => x.event.eventType.startsWith("tool.execution.") && rec(x.event.payload).transport === "builtin"); const write = native.filter(x => rec(x.event.payload).executionId === executionId);