From 9e0ede4cbe2f426f87b08f647885797ace9023da Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 1 Oct 2026 11:13:04 -0500 Subject: [PATCH] test(runner-e2e): retain Pi steering presentation evidence Co-Authored-By: Paperclip --- tests/runner-e2e/pi-controls-flow.test.ts | 39 +++++++++++++++++++++-- tests/runner-e2e/pi-controls-flow.ts | 25 ++++++++++++--- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/tests/runner-e2e/pi-controls-flow.test.ts b/tests/runner-e2e/pi-controls-flow.test.ts index 3c26f4c485c..13bc39b7d1c 100644 --- a/tests/runner-e2e/pi-controls-flow.test.ts +++ b/tests/runner-e2e/pi-controls-flow.test.ts @@ -5,6 +5,8 @@ import { expect, it, vi } from "vitest"; import { piControlTasks } from "./pi-controls-cases.js"; import { piControlFixture } from "./pi-controls-test-fixture.js"; import { runPiControlsFlow } from "./pi-controls-flow.js"; +import { persistedFinalRunMessage } from "./matchers.js"; +import { readPiSteeringSettlement } from "./pi-controls-evidence.js"; const harness = vi.hoisted(() => ({ target: "", prompt: "", message: "", mutation: false, incomplete: false })); vi.mock("./user-actions.js", () => ({ @@ -24,13 +26,15 @@ vi.mock("@playwright/test", () => ({ expect: (actual: any, message?: string) => toHaveCount: async (value: number) => { if (typeof actual.count === "function") expect(actual.count()).toBe(value); }, }) })); -async function exercise(taskId: string, remote: boolean, failure?: "mutation" | "incomplete" | "missing-ack") { +async function exercise(taskId: string, remote: boolean, failure?: "mutation" | "incomplete" | "missing-ack" | "missing-comment" | "foreign-comment") { harness.target = ""; harness.prompt = ""; harness.message = ""; harness.mutation = failure === "mutation"; harness.incomplete = failure === "incomplete"; const task = piControlTasks.find(t => t.id === taskId)!, stopCase = taskId === "pending-permission-stop"; const workspacePath = await mkdtemp(join(tmpdir(), "pi-controls-fixture-")); const saved = new Map(), localCleanup: Array<() => Promise> = [], remoteCleanup: Array<() => Promise> = []; let f = piControlFixture(`pi-control-fixture.txt`), stops = 0, staleDeclines = 0, browserSteers = 0, browserDeclines = 0; let steeringMarker = "", published = false, remoteSequence = 0; + let cleaningUp = false; + const publicComments: any[][] = []; const sync = () => { const target = remote ? "pi-control-fixture.txt" : harness.target; f.scope.target = target; f.tool.target = target; f.issue.title = task.buildTitle("fixture"); @@ -54,7 +58,15 @@ async function exercise(taskId: string, remote: boolean, failure?: "mutation" | if (path === "/api/heartbeat-runs/run") return f.run; if (path.includes("/events?")) return f.events; if (path.endsWith("/queued-comments")) return queue(); - if (path.endsWith("/comments")) return f.run.status === "succeeded" ? [{ id: "final", createdByRunId: "run", body: steeringMarker }] : []; + if (path.endsWith("/comments")) { + const comments = f.run.status !== "succeeded" ? [] : [ + { id: "decoy", createdByRunId: "other-run", body: steeringMarker, authorAgentId: "other-agent" }, + ...cleaningUp && failure === "missing-comment" ? [] : [{ id: "final", companyId: "company", issueId: "issue", + createdByRunId: cleaningUp && failure === "foreign-comment" ? "other-run" : "run", body: steeringMarker, + authorAgentId: "agent", createdAt: "2026-10-01T00:00:02Z", updatedAt: "2026-10-01T00:00:02Z", observedRead: publicComments.length }], + ]; + publicComments.push(structuredClone(comments)); return comments; + } throw new Error(`Unexpected read ${path}`); }, request: { post: async (path: string, options: any) => { @@ -84,6 +96,7 @@ async function exercise(taskId: string, remote: boolean, failure?: "mutation" | postBrowser("/api/issues/issue/queued-comments/comment/steer", { queueId: "queue", targetRunId: "run", revision: "revision" }); } else if (kind === "deny") { browserDeclines++; expect(browserSteers).toBe(1); expect(stopCase).toBe(false); f.finish(); + f.run.resultJson.presentationDecision = { commentId: "final", reason: "fixture-public-presentation" }; postBrowser("/api/heartbeat-runs/run/runtime-requests/request/resolve", { turnId: "turn", requestKind: "permission_approval", resolution: { action: "decline" } }); } else throw new Error(`Unexpected click ${kind}`); }, @@ -114,16 +127,35 @@ async function exercise(taskId: string, remote: boolean, failure?: "mutation" | observe: () => {}, capture: async () => {}, evidence: async (name: string, value: unknown) => saved.set(name, structuredClone(value)), remoteBootstrap, registerCleanupAssertion: (fn: () => Promise) => localCleanup.push(fn), registerBeforeEnvironmentTeardownAssertion: (fn: () => Promise) => remoteCleanup.push(fn) } as any); if (failure === "missing-ack") { await expect(call).rejects.toThrow("acknowledgement"); expect(browserDeclines).toBe(0); } - else if (remote && failure) await expect(call).rejects.toThrow(); + else if (remote && (failure === "mutation" || failure === "incomplete")) await expect(call).rejects.toThrow(); else { const result = await call; expect(result.checks.every(c => c.passed)).toBe(true); expect(saved.has("api-state.json")).toBe(true); expect(stops).toBe(stopCase ? 1 : 0); expect(staleDeclines).toBe(stopCase ? 1 : 0); expect(browserSteers).toBe(stopCase ? 0 : 1); expect(browserDeclines).toBe(stopCase ? 0 : 1); + if (!stopCase) { + const receipt = saved.get("pi-steering-presentation.json"); + expect(receipt).toMatchObject({ schema: "paperclip.e2e.pi-steering-presentation.v1", phase: "final", companyId: "company", issueId: "issue", runId: "run", commentsApiPath: "/api/issues/issue/comments" }); + expect(receipt.comments).toEqual(publicComments.at(-1)); + expect(receipt.run.resultJson.presentationDecision).toEqual(f.run.resultJson.presentationDecision); + expect(receipt).not.toHaveProperty("finalMessage"); + expect(readPiSteeringSettlement({ ...receipt, ...saved.get("api-state.json").steered, + finalMessage: persistedFinalRunMessage(receipt.comments, receipt.run) })).toEqual(saved.get("pi-steering-settlement.json")); + } } expect(remote ? localCleanup : remoteCleanup).toHaveLength(0); const cleanups = remote ? remoteCleanup : localCleanup; expect(cleanups).toHaveLength(1); + cleaningUp = true; if (failure) await expect(cleanups[0]!()).rejects.toThrow(); else expect((await cleanups[0]!())[0].passed).toBe(true); + if (!stopCase && (!failure || failure === "missing-comment" || failure === "foreign-comment")) { + const receipt = saved.get("pi-steering-presentation-after-cleanup.json"); + expect(receipt.phase).toBe("after-cleanup"); expect(receipt.comments).toEqual(publicComments.at(-1)); + expect(receipt.comments).not.toEqual(saved.get("pi-steering-presentation.json").comments); + const replay = () => readPiSteeringSettlement({ ...receipt, ...saved.get("api-state.json").steered, + finalMessage: persistedFinalRunMessage(receipt.comments, receipt.run) }); + if (failure) { expect(persistedFinalRunMessage(receipt.comments, receipt.run)).toBe(""); expect(replay).toThrow(); } + else expect(replay()).toEqual(saved.get("pi-steering-settlement.json")); + } if (remote) { expect(fixture.close).toHaveBeenCalledOnce(); expect(saved.get("pi-control-cleanup.json").retirement?.filesystemAfterRetirementObserved ?? false).toBe(false); } } finally { await rm(workspacePath, { recursive: true, force: true }); } } @@ -133,3 +165,4 @@ for (const task of ["pending-permission-stop", "same-turn-steering"]) { it.each(["mutation", "incomplete"] as const)("fails local cleanup on %s despite an absent final target", failure => exercise("pending-permission-stop", false, failure)); it.each(["mutation", "incomplete"] as const)("fails remote lifetime proof on %s despite an absent target", failure => exercise("same-turn-steering", true, failure)); it("never denies the native write before the steering acknowledgement exists", () => exercise("same-turn-steering", false, "missing-ack")); +it.each(["missing-comment", "foreign-comment"] as const)("retains and rejects %s during cleanup without fabricating persisted output", failure => exercise("same-turn-steering", false, failure)); diff --git a/tests/runner-e2e/pi-controls-flow.ts b/tests/runner-e2e/pi-controls-flow.ts index 6f5481b2017..d8fab26fef4 100644 --- a/tests/runner-e2e/pi-controls-flow.ts +++ b/tests/runner-e2e/pi-controls-flow.ts @@ -75,11 +75,26 @@ export async function runPiControlsFlow(input: { observeProcesses(); input.observe(issue, runs); return { run: runs[0] ?? {}, issue, events }; }; const scope = (): PiControlScope => ({ companyId: fixtures.company.id, issueId: issue.id, runId: runs[0]!.id, target }); - const finalMessage = async () => persistedFinalRunMessage(await api.get>(`/api/issues/${issue.id}/comments`), runs[0] as Row & { id: string }); - const assertSettled = async () => { + const readPresentation = async (state: PiControlState) => { + const commentsApiPath = `/api/issues/${state.issue.id}/comments`; + const comments = await api.get>(commentsApiPath); + return { ...state, commentsApiPath, comments }; + }; + const finalMessage = (presentation: Awaited>) => persistedFinalRunMessage(presentation.comments, presentation.run as Row & { id: string }); + const assertSettled = async (phase: "final" | "after-cleanup" = "final") => { const state = await load(); if (stopped) return readPiStopSettlement({ ...state, ...stopped }); - if (steered) return readPiSteeringSettlement({ ...state, ...steered, finalMessage: await finalMessage() }); + if (steered) { + const presentation = await readPresentation(state); + // Retain the public source records, including the run's presentation + // decision, before grading. A marker or derived string cannot replace + // the persisted comment selected by the Product. + await input.evidence(`pi-steering-presentation${phase === "after-cleanup" ? "-after-cleanup" : ""}.json`, { + schema: "paperclip.e2e.pi-steering-presentation.v1", phase, + companyId: fixtures.company.id, issueId: state.issue.id, runId: state.run.id, ...presentation, + }); + return readPiSteeringSettlement({ ...presentation, ...steered, finalMessage: finalMessage(presentation) }); + } throw new Error("Pi controls settlement is missing"); }; const sample = async (phase: string) => { @@ -109,7 +124,7 @@ export async function runPiControlsFlow(input: { retirement = assertActiveStopRetirement({ completed, identityChanged: processError, processes, watcher: remote ? sealed!.watcher : local!.watcher.finish(), ...(remote ? { environment: "daytona" as const, remote: { scope: scope(), observations: remoteObservations } } : { environment: "local" as const, samples: samples.map(s => ({ ...s, phase: s.phase === "after-control" ? "after-stop" : s.phase })) }) }); - await assertSettled(); + await assertSettled("after-cleanup"); return [{ id: "pi-control-retired-no-effects", passed: true, detail: remote ? "Owned remote tree retired with continuous no-effects proof through its lifetime seal" : "Owned local tree retired; no target mutation through cleanup" }]; } finally { if (timer) clearInterval(timer); @@ -181,7 +196,7 @@ export async function runPiControlsFlow(input: { const decision = (await declined).postDataJSON(); check("exact-browser-decline", decision.turnId === pending.turnId && decision.requestKind === "permission_approval" && decision.resolution?.action === "decline", "Browser denied the still-pending original native write"); await pollUntil({ label: "Pi consumed same-turn steering", deadlineAt: input.deadlineAt, intervalMs: 200, - load: async () => { const state = await load(); if (["cancelled", "failed", "timed_out"].includes(state.run.status)) throw new Error("Stopped waiting for Pi consumed same-turn steering: provider failed"); return { ...state, finalMessage: await finalMessage() }; }, + load: async () => { const state = await load(); if (["cancelled", "failed", "timed_out"].includes(state.run.status)) throw new Error("Stopped waiting for Pi consumed same-turn steering: provider failed"); const presentation = await readPresentation(state); return { ...state, finalMessage: finalMessage(presentation) }; }, accept: state => { readPiSteeringSettlement({ ...state, ...steered! }); return true; } }); await input.evidence("pi-steering-settlement.json", await assertSettled()); }