fix(evals): scope compact reports and retain setup privacy audits

tests/runner-e2e/README.md

@@ -636,8 +636,11 @@ inputs, or a tracked env file.
 
 ## Private framework evidence
 
-An approved private-cell preflight uses Playwright's official JSON and JUnit
-reporters. Other runs keep the existing HTML and blob reporters. The immutable
+The protected TestOps reporter (`PAPERCLIP_CHAT_COMPLETION_TESTOPS=v1`) uses
+Playwright's official JSON and JUnit reporters only with an approved private-cell
+preflight. Other runs, including generic private full-stack runs, keep the existing
+HTML and blob reporters. A setup failure can retain its SDK audit without a
+framework report; a malformed present framework report still fails closed. The immutable
 JSON source is retained in `snapshots/playwright-framework-original.txt` and is
 parsed and deeply inspected as registered Playwright JSON, including decoded
 inline text/JSON attachment bodies. The text suffix preserves source bytes; it

tests/runner-e2e/playwright.config.ts

@@ -81,7 +81,7 @@ export default defineConfig({
     stderr: "pipe",
   },
   outputDir: path.join(privateDir, "playwright-output"),
-  reporter: process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT ? [
+  reporter: process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT && process.env.PAPERCLIP_CHAT_COMPLETION_TESTOPS === "v1" ? [
     ["list"],
     ["json", { outputFile: path.join(privateDir, "framework-report.json") }],
     ["junit", { outputFile: path.join(privateDir, "junit.xml") }],

tests/runner-e2e/private-framework-report-mode.test.ts

@@ -0,0 +1,16 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { buildRunnerE2EProcessEnvironment } from "./harness-env.js";
+afterEach(() => { vi.unstubAllEnvs(); vi.resetModules(); });
+it.each([
+  [undefined, undefined, ["list", "blob", "junit", "html"]],
+  ["/tmp/owned/preflight.json", undefined, ["list", "blob", "junit", "html"]],
+  [undefined, "v1", ["list", "blob", "junit", "html"]],
+  ["/tmp/owned/preflight.json", "v1", ["list", "json", "junit"]],
+] as const)("selects supported private reporter mode for preflight=%s/TestOps=%s", async (preflight, testops, expected) => {
+  const source = { PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: preflight, PAPERCLIP_CHAT_COMPLETION_TESTOPS: testops };
+  expect(buildRunnerE2EProcessEnvironment(source, []).PAPERCLIP_CHAT_COMPLETION_TESTOPS).toBe(testops);
+  for (const [key, value] of Object.entries({ ...source, PAPERCLIP_RUNNER_E2E_PORT: "3104", PAPERCLIP_RUNNER_E2E_TEMP_ROOT: "/tmp/owned", PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: "/tmp/owned/private",
+    PAPERCLIP_HOME: "/tmp/owned/home", PAPERCLIP_CONFIG: "/tmp/owned/config.json", PAPERCLIP_INSTANCE_ID: "synthetic", PAPERCLIP_AGENT_JWT_SECRET: "synthetic", PAPERCLIP_DECISION_SIGNING_SECRET: "synthetic", PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: "synthetic", BETTER_AUTH_SECRET: "synthetic" })) vi.stubEnv(key, value);
+  const { default: config } = await import("./playwright.config.js");
+  expect((config.reporter as Array<[string]>).map(row => row[0])).toEqual(expected);
+});

tests/runner-e2e/shared-evidence.ts

@@ -1,6 +1,6 @@
 import { copyPrivateFrameworkReport } from "./private-framework-report.js";
 import type { SourceSdkAuditCapture } from "./source-sdk-evidence.js";
-import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
+import { cp, lstat, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { createHash } from "node:crypto";
 import { runnerE2EServerControlPaths } from "./harness-env.js";
@@ -24,7 +24,9 @@ export async function copySharedEvidence(privateRoot: string, casePrivateDir: st
       if (error.code !== "ENOENT") throw error;
     });
   }
-  if (sdkAudit) await copyPrivateFrameworkReport(privateRoot, casePrivateDir, secrets);
+  const frameworkReport = sdkAudit && await lstat(path.join(privateRoot, "framework-report.json")).then(() => true,
+    (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return false; throw error; });
+  if (frameworkReport) await copyPrivateFrameworkReport(privateRoot, casePrivateDir, secrets);
   if (sdkAudit) await (await import("./source-sdk-evidence.js")).copyPreparedSourceSdkAudit(temporaryRoot, casePrivateDir, secrets, sdkAudit);
   // The supervisor records each phase before it can fail. Copy its receipts
   // independently of the successful-restart acknowledgement before cleanup.

tests/runner-e2e/source-sdk-audit-capture.test.ts

@@ -53,3 +53,8 @@ it("keeps the captured audit immutable and leaves non-SDK/missing-audit paths un
  await expect(f.reject()).rejects.toThrow("Source SDK evidence privacy check failed");await f.copy();const original=await readFile(f.destination);
  await expect(f.copy()).rejects.toThrow();expect(await readFile(f.destination)).toEqual(original);
 });
+
+it("rejects a malformed present framework report instead of hiding it as a missing setup report",async()=>{
+ const f=await fixture();await writeFile(path.join(f.capture,"framework-report.json"),"{");
+ await expect(f.copy()).rejects.toThrow();
+});