import { execFileSync } from "node:child_process"; import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { piNativeTasks } from "./pi-native-cases.js"; import { packageEvidence } from "./evidence.js"; import { runPiNativeFlow } from "./pi-native-flow.js"; import { sanitizeJson } from "./redaction.js"; // Drive the actual flow and packager without a browser, provider, or server. // Only the external browser/API boundaries are scripted. vi.mock("./user-actions.js", () => ({ createTaskThroughUi: vi.fn(async (input: { requireExplicitTitle?: boolean }) => { expect(input.requireExplicitTitle).toBe(true); return { submittedAtMs: Date.now(), issueId: "issue-fixture" }; }), })); vi.mock("@playwright/test", () => ({ expect: (actual: unknown, message?: string) => ({ toBe: (expected: unknown) => expect(actual, message).toBe(expected), toBeVisible: async () => expect(actual, message).toEqual({ visible: true }), }), })); const roots: string[] = []; afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); type FlowInput = Parameters[0]; async function fixture(options: { failComments?: boolean; remote?: boolean; incompleteRemote?: boolean } = {}) { const root = await mkdtemp(join(tmpdir(), "pi-native-evidence-")); roots.push(root); const privateDir = join(root, "private"); const uploadDir = join(root, "upload"); const workspacePath = join(root, "workspace"); const snapshots = join(privateDir, "snapshots"); for (const directory of [workspacePath, snapshots, join(privateDir, "html-report"), join(privateDir, "blob-report")]) await mkdir(directory, { recursive: true }); const execution = { id: "pi-native.runner-acpx-pi.local.native-questions", environment: { id: options.remote ? "daytona" : "local" }, profile: { qualificationCandidate: "pi" }, task: piNativeTasks.find(row => row.id === "native-questions")! } as FlowInput["execution"]; const issue = { id: "issue-fixture", identifier: "PI-1", title: "Provider-generated task name", companyId: "company-fixture", assigneeAgentId: "agent-fixture", status: "done" }; const run = { id: "run-fixture", status: "succeeded", runtimeMode: "native", createdAt: "2026-09-29T00:00:00Z" }; const headers = ["Pi native color", "Pi native confirmation", "Pi native name", "Pi native draft"]; let answered = 0; const typed: string[] = []; let answerProof = ""; let remoteFinished = false; const cleanupAssertions: Array<() => Promise> = []; const interactions = () => headers.slice(0, Math.min(4, answered + 1)).map((header, index) => ({ id: `question-${index}`, kind: "ask_user_questions", status: index < answered ? "answered" : "pending", sourceRunId: run.id, continuationPolicy: "none", payload: { runtimeRequestId: `native-${index}`, questionSet: { questions: [{ header }] } }, })); const secret = "private-fixture-sentinel-value"; const comments = [{ id: "comment-fixture", body: `Completed ${secret}` }]; // A full first page requires the real collector to retain a second page. const eventRows = Array.from({ length: 1001 }, (_, index) => ({ seq: index + 1, eventType: "fixture" })); const paths: string[] = []; const api = { post: async () => ({ name: "Pi fixture project" }), get: async (path: string) => { paths.push(path); if (path === "/api/companies/company-fixture/issues?limit=100") return [issue]; if (path === `/api/issues/${issue.id}`) return issue; if (path === "/api/companies/company-fixture/heartbeat-runs?limit=100") return [run]; if (path === `/api/heartbeat-runs/${run.id}`) return run; if (path === `/api/issues/${issue.id}/interactions`) return interactions(); if (path === `/api/issues/${issue.id}/comments`) { if (options.failComments) throw new Error("fixture comments unavailable"); return comments; } if (path.startsWith(`/api/heartbeat-runs/${run.id}/events?`)) { const afterSeq = Number(new URL(path, "http://fixture.invalid").searchParams.get("afterSeq")); return eventRows.filter(row => row.seq > afterSeq).slice(0, 1000); } throw new Error(`Unexpected fixture API path ${path}`); }, }; const page = { goto: async () => {}, reload: async () => {}, getByTestId: (id: string) => id === "issue-detail-header" ? { getByRole: () => ({ visible: true }) } : { last: () => ({ locator: () => ({ first: () => ({ fill: async (value: string) => { typed.push(value); } }) }) }) }, getByRole: (role: string) => ({ last: () => ({ click: async () => { if (role !== "button") return; answered += 1; if (answered === 4) { answerProof = JSON.stringify([ { status: "answered", optionId: "blue" }, { status: "negative_or_cancelled", confirmed: false }, { status: "answered", value: typed[0] }, { status: "answered", value: typed[1] }, ]); await writeFile(join(workspacePath, "pi-native-answers.json"), options.remote ? "WRONG HOST COPYBACK" : answerProof); } } }) }), }; const evidence = async (name: string, data: unknown) => { await writeFile(join(snapshots, name), JSON.stringify(sanitizeJson(data, [secret]))); }; const capture = vi.fn(async (_id: string, _label: string, file: string) => { await writeFile(join(privateDir, file), "fixture screenshot boundary"); }); const remoteRoot = { pid: 10, ppid: 1, startTicks: "100", bootId: "boot" }; const remoteSnapshot = { observedAtMs: 2, complete: !options.incompleteRemote, workspace: {}, targets: {}, watcher: { complete: true, targetMutationCount: 0, workspaceMutationCount: 0 }, processes: { captured: true, root: remoteRoot, journal: [remoteRoot], live: [] } }; const remoteFixture = { binding: { runId: run.id }, remoteCwd: "/owned/remote", outsideTarget: null, snapshot: async () => remoteSnapshot, finish: async () => { remoteFinished = true; return remoteSnapshot; }, readFile: async (path: string) => { expect(remoteFinished).toBe(true); expect(path).toBe("pi-native-answers.json"); return Buffer.from(answerProof); }, close: vi.fn(async () => {}), }; const input = { page: page as unknown as FlowInput["page"], api: api as unknown as FlowInput["api"], fixtures: { company: { id: "company-fixture", issuePrefix: "PI" }, agent: { id: "agent-fixture", name: "Pi fixture" }, environment: { id: "environment-fixture" } } as FlowInput["fixtures"], execution, nonce: "fixture", workspacePath, deadlineAt: Date.now() + 5000, restart: async () => { throw new Error("Unexpected restart"); }, observe: () => {}, capture, evidence, ...(options.remote ? { registerCleanupAssertion: (fn: () => Promise) => cleanupAssertions.push(fn), remoteBootstrap: { prompt: () => "Read the bootstrap file only", bindAndRelease: async (value: { targets: string[]; actionPrompt(fixture: any): Promise | string }) => { expect(value.targets).toEqual(["pi-native-answers.json"]); const prompt = await value.actionPrompt(remoteFixture); expect(prompt).toContain("paperclip_native_question"); return remoteFixture; }, } } : {}), }; for (const [file, content] of [["result.json", "{}"], ["snapshots/fixtures.json", "{}"], ["junit.xml", ""], ["html-report/index.html", "fixture report"]]) await writeFile(join(privateDir, file!), content!); const blob = join(privateDir, "blob-report"); await writeFile(join(blob, "fixture.txt"), "unit fixture"); execFileSync("zip", ["-q", "report.zip", "fixture.txt"], { cwd: blob }); return { input: input as FlowInput, privateDir, uploadDir, snapshots, paths, capture, issue, run, secret, cleanupAssertions }; } describe("Pi native terminal evidence", () => { it("packages the actual successful native-question flow with durable API state and every event page", async () => { const f = await fixture(); const result = await runPiNativeFlow(f.input); expect(result.checks).toHaveLength(16); expect(result.checks.every(check => check.passed)).toBe(true); const packaged = await packageEvidence({ privateDir: f.privateDir, uploadDir: f.uploadDir, secrets: [f.secret], expectPassScreenshot: true }); expect(packaged.missing).toEqual([]); expect(packaged.leaks).toEqual([]); const raw = await readFile(join(f.uploadDir, "snapshots/api-state.json"), "utf8"); expect(raw).not.toContain(f.secret); const state = JSON.parse(raw); expect(state.issue).toEqual(f.issue); expect(state.runs).toEqual([f.run]); expect(state.run).toEqual(f.run); expect(state.comments).toHaveLength(1); expect(state.comments[0].id).toBe("comment-fixture"); expect(state.interactions).toHaveLength(4); expect(state.interactions.every((row: { status: string }) => row.status === "answered")).toBe(true); expect(state.runEventsByRun[0].runId).toBe(f.run.id); expect(state.runEventsByRun[0].events).toHaveLength(1001); expect(state.runEventsByRun[0].events.at(-1).seq).toBe(1001); expect(f.paths).toContain(`/api/heartbeat-runs/${f.run.id}/events?afterSeq=1000&limit=1000`); // The original failure remains a failure: Pi-specific evidence cannot // substitute for the mandatory generic terminal API snapshot. await rm(join(f.snapshots, "api-state.json")); const incomplete = await packageEvidence({ privateDir: f.privateDir, uploadDir: f.uploadDir, secrets: [f.secret], expectPassScreenshot: true }); expect(incomplete.missing).toEqual(["snapshots/api-state.json"]); }); it("fails capture when terminal API evidence is unavailable, retaining behavioral checks without declaring success", async () => { const f = await fixture({ failComments: true }); await expect(runPiNativeFlow(f.input)).rejects.toThrow("fixture comments unavailable"); const checks = JSON.parse(await readFile(join(f.snapshots, "pi-native-checks.json"), "utf8")); expect(checks.checks).toHaveLength(16); expect(checks.checks.every((check: { passed: boolean }) => check.passed)).toBe(true); expect(f.capture.mock.calls.some(([id]) => id === "final-state")).toBe(false); const packaged = await packageEvidence({ privateDir: f.privateDir, uploadDir: f.uploadDir, secrets: [f.secret], expectPassScreenshot: true }); expect(packaged.missing).toEqual(["final-state.png", "snapshots/api-state.json"]); }); }); it("uses sealed remote answer bytes rather than host copyback and requires independent retirement", async () => { const f = await fixture({ remote: true }); const result = await runPiNativeFlow(f.input); expect(result.checks.every(check => check.passed)).toBe(true); expect(f.cleanupAssertions).toHaveLength(1); await f.cleanupAssertions[0]!(); const retained = JSON.parse(await readFile(join(f.snapshots, "pi-remote-1-questions-final.json"), "utf8")); expect(retained.snapshot.processes.live).toEqual([]); }); it("cannot replace missing remote retirement proof with a succeeded Product run", async () => { const f = await fixture({ remote: true, incompleteRemote: true }); await expect(runPiNativeFlow(f.input)).rejects.toThrow("retirement is unproven"); await expect(f.cleanupAssertions[0]!()).rejects.toThrow("retirement proof incomplete"); });