From 006a5c19712cb844bd44645648ee5c3ca70a0cb3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 17:28:30 -0600 Subject: [PATCH] fix(evals): scope compact reports and retain setup privacy audits --- tests/runner-e2e/README.md | 7 +++++-- tests/runner-e2e/playwright.config.ts | 2 +- .../private-framework-report-mode.test.ts | 16 ++++++++++++++++ tests/runner-e2e/shared-evidence.ts | 6 ++++-- .../runner-e2e/source-sdk-audit-capture.test.ts | 5 +++++ 5 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 tests/runner-e2e/private-framework-report-mode.test.ts diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 1a030d51567..7faa0608e49 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -636,8 +636,11 @@ inputs, or a tracked env file. ## Private framework evidence -An approved private-cell preflight uses Playwright's official JSON and JUnit -reporters. Other runs keep the existing HTML and blob reporters. The immutable +The protected TestOps reporter (`PAPERCLIP_CHAT_COMPLETION_TESTOPS=v1`) uses +Playwright's official JSON and JUnit reporters only with an approved private-cell +preflight. Other runs, including generic private full-stack runs, keep the existing +HTML and blob reporters. A setup failure can retain its SDK audit without a +framework report; a malformed present framework report still fails closed. The immutable JSON source is retained in `snapshots/playwright-framework-original.txt` and is parsed and deeply inspected as registered Playwright JSON, including decoded inline text/JSON attachment bodies. The text suffix preserves source bytes; it diff --git a/tests/runner-e2e/playwright.config.ts b/tests/runner-e2e/playwright.config.ts index 9cf4f4c5d9d..6ba9419b5df 100644 --- a/tests/runner-e2e/playwright.config.ts +++ b/tests/runner-e2e/playwright.config.ts @@ -81,7 +81,7 @@ export default defineConfig({ stderr: "pipe", }, outputDir: path.join(privateDir, "playwright-output"), - reporter: process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT ? [ + reporter: process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT && process.env.PAPERCLIP_CHAT_COMPLETION_TESTOPS === "v1" ? [ ["list"], ["json", { outputFile: path.join(privateDir, "framework-report.json") }], ["junit", { outputFile: path.join(privateDir, "junit.xml") }], diff --git a/tests/runner-e2e/private-framework-report-mode.test.ts b/tests/runner-e2e/private-framework-report-mode.test.ts new file mode 100644 index 00000000000..63eacd349c6 --- /dev/null +++ b/tests/runner-e2e/private-framework-report-mode.test.ts @@ -0,0 +1,16 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; +afterEach(() => { vi.unstubAllEnvs(); vi.resetModules(); }); +it.each([ + [undefined, undefined, ["list", "blob", "junit", "html"]], + ["/tmp/owned/preflight.json", undefined, ["list", "blob", "junit", "html"]], + [undefined, "v1", ["list", "blob", "junit", "html"]], + ["/tmp/owned/preflight.json", "v1", ["list", "json", "junit"]], +] as const)("selects supported private reporter mode for preflight=%s/TestOps=%s", async (preflight, testops, expected) => { + const source = { PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: preflight, PAPERCLIP_CHAT_COMPLETION_TESTOPS: testops }; + expect(buildRunnerE2EProcessEnvironment(source, []).PAPERCLIP_CHAT_COMPLETION_TESTOPS).toBe(testops); + for (const [key, value] of Object.entries({ ...source, PAPERCLIP_RUNNER_E2E_PORT: "3104", PAPERCLIP_RUNNER_E2E_TEMP_ROOT: "/tmp/owned", PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: "/tmp/owned/private", + PAPERCLIP_HOME: "/tmp/owned/home", PAPERCLIP_CONFIG: "/tmp/owned/config.json", PAPERCLIP_INSTANCE_ID: "synthetic", PAPERCLIP_AGENT_JWT_SECRET: "synthetic", PAPERCLIP_DECISION_SIGNING_SECRET: "synthetic", PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: "synthetic", BETTER_AUTH_SECRET: "synthetic" })) vi.stubEnv(key, value); + const { default: config } = await import("./playwright.config.js"); + expect((config.reporter as Array<[string]>).map(row => row[0])).toEqual(expected); +}); diff --git a/tests/runner-e2e/shared-evidence.ts b/tests/runner-e2e/shared-evidence.ts index d89d7adc7dc..ba72dcbf87c 100644 --- a/tests/runner-e2e/shared-evidence.ts +++ b/tests/runner-e2e/shared-evidence.ts @@ -1,6 +1,6 @@ import { copyPrivateFrameworkReport } from "./private-framework-report.js"; import type { SourceSdkAuditCapture } from "./source-sdk-evidence.js"; -import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { cp, lstat, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { createHash } from "node:crypto"; import { runnerE2EServerControlPaths } from "./harness-env.js"; @@ -24,7 +24,9 @@ export async function copySharedEvidence(privateRoot: string, casePrivateDir: st if (error.code !== "ENOENT") throw error; }); } - if (sdkAudit) await copyPrivateFrameworkReport(privateRoot, casePrivateDir, secrets); + const frameworkReport = sdkAudit && await lstat(path.join(privateRoot, "framework-report.json")).then(() => true, + (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return false; throw error; }); + if (frameworkReport) await copyPrivateFrameworkReport(privateRoot, casePrivateDir, secrets); if (sdkAudit) await (await import("./source-sdk-evidence.js")).copyPreparedSourceSdkAudit(temporaryRoot, casePrivateDir, secrets, sdkAudit); // The supervisor records each phase before it can fail. Copy its receipts // independently of the successful-restart acknowledgement before cleanup. diff --git a/tests/runner-e2e/source-sdk-audit-capture.test.ts b/tests/runner-e2e/source-sdk-audit-capture.test.ts index 3f3cc13faeb..34cba77168a 100644 --- a/tests/runner-e2e/source-sdk-audit-capture.test.ts +++ b/tests/runner-e2e/source-sdk-audit-capture.test.ts @@ -53,3 +53,8 @@ it("keeps the captured audit immutable and leaves non-SDK/missing-audit paths un await expect(f.reject()).rejects.toThrow("Source SDK evidence privacy check failed");await f.copy();const original=await readFile(f.destination); await expect(f.copy()).rejects.toThrow();expect(await readFile(f.destination)).toEqual(original); }); + +it("rejects a malformed present framework report instead of hiding it as a missing setup report",async()=>{ + const f=await fixture();await writeFile(path.join(f.capture,"framework-report.json"),"{"); + await expect(f.copy()).rejects.toThrow(); +});