test(evals): retain closed packaging budget diagnostics

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/evidence.ts

@@ -71,9 +71,32 @@ export function isStandardEvidenceFile(relative: string) {
     || extension === ".zip");
 }
 
+// Diagnostic metadata only: never an admission, completeness or grading receipt.
+// inventoryComplete describes the listed standard files, not a sealed original tree.
+interface PrivateEvidenceBudget {
+  bound: string; listedFiles: number; filesExamined: number; bytesExamined: number;
+  inventoryComplete: boolean; scanFinished: boolean; requiredEvidenceComplete: boolean | null;
+  byKind: Record<string, { files: number; bytes: number }>;
+}
+const evidenceBudgetKinds = ["json", "log", "image", "video", "zip", "html", "text", "asset"];
+function evidenceBudgetKind(extension: string): string {
+  return extension === ".json" ? "json" : extension === ".log" ? "log" : [".png", ".jpg", ".jpeg", ".gif", ".webp"].includes(extension) ? "image"
+    : [".webm", ".mp4"].includes(extension) ? "video" : extension === ".zip" ? "zip" : extension === ".html" ? "html"
+    : [".txt", ".md", ".xml"].includes(extension) ? "text" : "asset";
+}
+function budgetFailure(message: string, budget: PrivateEvidenceBudget, bound: string) {
+  budget.bound = bound;
+  return Object.assign(new Error(message), { evidenceBudget: budget });
+}
+function scannerBudgetBound(reason: string): string {
+  return reason === "zip inspection exceeded its expanded-byte bound" ? "zip-expanded-bytes"
+    : reason === "zip inspection exceeded its time bound" ? "zip-time"
+    : reason === "embedded report exceeded its byte bound" ? "html-encoded-bytes" : "none";
+}
 export interface EvidencePackageResult {
   files: string[];
   leaks: Array<{ file: string; reason: string }>;
+  budget?: PrivateEvidenceBudget;
   missing: string[];
 }
 
@@ -186,7 +209,10 @@ export async function packageEvidence(input: {
   const available = await walk(input.privateDir, "", Boolean(input.privateCell));
   const standard = available.filter(isStandardEvidencePath);
 
-  if (input.privateCell && standard.length > 4096) throw new Error("Private cell evidence exceeds entry allowance");
+  const budget: PrivateEvidenceBudget | undefined = input.privateCell ? { bound: "none", listedFiles: Math.min(standard.length, 4097), filesExamined: 0, bytesExamined: 0,
+    inventoryComplete: false, scanFinished: false, requiredEvidenceComplete: null,
+    byKind: Object.fromEntries(evidenceBudgetKinds.map(kind => [kind, { files: 0, bytes: 0 }])) } : undefined;
+  if (input.privateCell && standard.length > 4096) throw budgetFailure("Private cell evidence exceeds entry allowance", budget!, "entries");
   const privateSecrets = input.privateCell ? normalizedSecrets(input.secrets).flatMap(value => [value, encodeURIComponent(value), Buffer.from(value).toString("base64"), Buffer.from(value).toString("base64url")]) : [];
   let privateBytes = 0;
   for (const relative of standard) {
@@ -196,8 +222,12 @@ export async function packageEvidence(input: {
     if (input.privateCell) {
       assertSecretFree(relative, privateSecrets, "private cell evidence path");
       const metadata = await lstat(source);
+      if (Number.isSafeInteger(metadata.size) && metadata.size >= 0 && Number.isSafeInteger(budget!.bytesExamined + metadata.size)) {
+        budget!.filesExamined++; budget!.bytesExamined += metadata.size;
+        const counter = budget!.byKind[evidenceBudgetKind(extension)]!; counter.files++; counter.bytes += metadata.size;
+      }
       if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > 256 * 1024 * 1024 - privateBytes)
-        throw new Error("Private cell evidence exceeds byte allowance");
+        throw budgetFailure("Private cell evidence exceeds byte allowance", budget!, "physical-bytes");
       privateBytes += metadata.size;
       if (!Number.isSafeInteger(privateBytes)) throw new Error("Invalid private cell evidence byte accounting");
       const asset = cellTraceAsset(relative, input.privateCell.owner, input.privateCell.layout);
@@ -218,7 +248,7 @@ export async function packageEvidence(input: {
       const raw = await readFile(source, "utf8");
       if (extension === ".html") {
         const issue = await inspectEmbeddedReport(raw, input.secrets);
-        if (issue) { leaks.push({ file: relative, reason: issue }); continue; }
+        if (issue) { if (budget && budget.bound === "none") budget.bound = scannerBudgetBound(issue); leaks.push({ file: relative, reason: issue }); continue; }
       }
       const parsed = extension === ".json" ? JSON.parse(raw) : null;
       const leak =
@@ -248,6 +278,7 @@ export async function packageEvidence(input: {
     } else if (extension === ".zip") {
       const leak = await inspectZip(source, input.secrets);
       if (leak) {
+        if (budget && budget.bound === "none") budget.bound = scannerBudgetBound(leak);
         leaks.push({ file: relative, reason: leak });
         continue;
       }
@@ -269,6 +300,7 @@ export async function packageEvidence(input: {
     }
   }
 
+  if (budget) { budget.inventoryComplete = true; budget.scanFinished = true; }
   const missing = input.expectPassScreenshot
     ? [
         ...REQUIRED_PASS_FILES.filter((required) => !files.includes(required)),
@@ -299,7 +331,8 @@ export async function packageEvidence(input: {
     "utf8",
   );
   files.push("evidence-manifest.json");
-  return { files, leaks, missing };
+  if (budget && input.expectPassScreenshot) budget.requiredEvidenceComplete = missing.length === 0 && leaks.length === 0;
+  return { files, leaks, missing, ...(budget ? { budget } : {}) };
 }
 
 export async function publishRunnerE2EAttemptEvidence(input: {

tests/runner-e2e/private-attempt-diagnostic.ts

@@ -3,6 +3,33 @@ import { lstat, open, readFile, realpath } from "node:fs/promises";
 import path from "node:path";
 import { PAPERCLIP_PROTOCOL_ACTIONS } from "../../packages/paperclip-runner/src/protocol-actions/index.js";
 
+// Reject malformed counters wholesale. Unknown input never becomes a diagnostic
+// string, path, or authority; even a complete scan remains unqualified evidence.
+function closedPackagingBudget(input: unknown, message: string) {
+  const b = input && typeof input === "object" ? input as Record<string, any> : {};
+  const kinds = ["json", "log", "image", "video", "zip", "html", "text", "asset"];
+  const number = (v: unknown) => Number.isSafeInteger(v) && (v as number) >= 0;
+  if (![b.listedFiles, b.filesExamined, b.bytesExamined].every(number) || b.listedFiles > 4097 || b.filesExamined > 4096
+    || b.filesExamined > b.listedFiles || typeof b.inventoryComplete !== "boolean" || typeof b.scanFinished !== "boolean"
+    || ![true, false, null].includes(b.requiredEvidenceComplete)) return null;
+  const byKind: Record<string, { files: number; bytes: number }> = {};
+  for (const kind of kinds) {
+    const row = b.byKind?.[kind];
+    if (!row || !number(row.files) || !number(row.bytes)) return null;
+    byKind[kind] = { files: row.files, bytes: row.bytes };
+  }
+  if (Object.values(byKind).reduce((n, r) => n + r.files, 0) !== b.filesExamined
+    || Object.values(byKind).reduce((n, r) => n + r.bytes, 0) !== b.bytesExamined
+    || b.inventoryComplete && b.filesExamined !== b.listedFiles || b.scanFinished && !b.inventoryComplete) return null;
+  if (b.requiredEvidenceComplete === true && (!b.scanFinished || b.bound !== "none")) return null;
+  const observedBound = ["none", "physical-bytes", "entries", "zip-expanded-bytes", "zip-time", "html-encoded-bytes"].includes(b.bound) ? b.bound : "unknown";
+  const bound = message === "Complete original evidence exceeds byte allowance" ? "original-seal-bytes"
+    : message === "Complete original evidence exceeds entry allowance" ? "original-seal-entries"
+    : message === "Complete packaged evidence exceeds allowance" ? "final-seal-allowance" : observedBound;
+  return { scope: "package-standard-files", bound, listedFiles: b.listedFiles, filesExamined: b.filesExamined, bytesExamined: b.bytesExamined,
+    inventoryComplete: b.inventoryComplete, scanFinished: b.scanFinished, requiredEvidenceComplete: b.requiredEvidenceComplete,
+    byKind, qualification: "unqualified" };
+}
 export interface PrivateAttemptDiagnosticInput { stage: unknown; result?: unknown; error?: unknown; observation?: unknown; evidence?: unknown }
 const record = (value: unknown): Record<string, any> => value && typeof value === "object" ? value as Record<string, any> : {};
 const member = (value: unknown, choices: string[]) => typeof value === "string" && choices.includes(value) ? value : "unknown";
@@ -71,6 +98,7 @@ export function privateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput) {
   const evidence = record(input.evidence), leaks = Array.isArray(evidence.leaks) ? evidence.leaks : [];
   return {
     schema: "paperclip.private-attempt-diagnostic/v1",
+    packagingBudget: closedPackagingBudget(record(input.error).evidenceBudget ?? evidence.budget, message),
     stage: member(input.stage, ["execution", "package-original", "seal-original"]),
     status: member(result.status, ["passed", "failed", "skipped"]),
     failureClass: member(result.failureClass, ["candidate_failure", "transient_infrastructure", "permanent_infrastructure", "secret_leak", "cleanup_failure"]),

tests/runner-e2e/private-evidence-budget.test.ts

@@ -0,0 +1,68 @@
+import { mkdtemp, mkdir, truncate, writeFile, readFile, rm, realpath, symlink } from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import { execFileSync } from "node:child_process";
+import { afterEach, expect, it } from "vitest";
+import { packageEvidence } from "./evidence.js";
+import { inspectCellReportProducer, privateCellEvidenceOwner } from "./manual-private-report-assets.js";
+import { appendPrivateAttemptDiagnostic, privateAttemptDiagnostic } from "./private-attempt-diagnostic.js";
+const roots: string[] = [];
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, {recursive:true,force:true}))); });
+async function fixture() {
+ const root=await mkdtemp(path.join(await realpath(os.tmpdir()),"closed-budget-"));roots.push(root);
+ const cell=path.join(root,"capture/cases/activity.agent-feedback");await mkdir(cell,{recursive:true});
+ const id="chat-features.runner-opencode.local.activity.agent-feedback";
+ const owner=await privateCellEvidenceOwner({root,private:true,runId:"123",attempt:"1",executionIds:[id],cellReportProducer:await inspectCellReportProducer()},id);
+ return {root,cell,input:{privateDir:cell,uploadDir:path.join(root,"upload"),secrets:[],expectPassScreenshot:true,privateCell:{owner,layout:"original" as const}}};
+}
+it("a real sparse oversized source still rejects, but its closed byte witness survives the existing runner diagnostic path",async()=>{
+ const f=await fixture();const source=path.join(f.cell,"result.json");await writeFile(source,"");await truncate(source,256*1024*1024+1);
+ const error=await packageEvidence(f.input).then(()=>null,error=>error);expect(error?.message).toBe("Private cell evidence exceeds byte allowance");
+ const d=privateAttemptDiagnostic({stage:"package-original",result:{status:"passed"},error});
+ expect(d).toMatchObject({finalization:"byte-bound",packagingBudget:{bound:"physical-bytes",inventoryComplete:false,scanFinished:false,requiredEvidenceComplete:null,qualification:"unqualified",filesExamined:1,bytesExamined:268435457,byKind:{json:{files:1,bytes:268435457}}}});
+ const commands=path.join(f.root,"_runner_file_commands");await mkdir(commands);const envFile=path.join(commands,"set_env_budget");await writeFile(envFile,"");
+ await appendPrivateAttemptDiagnostic({stage:"package-original",result:{status:"passed"},error},{GITHUB_ACTIONS:"true",RUNNER_TEMP:f.root,GITHUB_ENV:envFile});
+ const wire=await readFile(envFile,"utf8");expect(wire).toContain('"bound":"physical-bytes"');expect(wire).not.toContain(source);expect(wire).not.toContain("closed-budget-");expect(wire.trim().split("\n")).toHaveLength(1);
+});
+
+it("retains the real ZIP expansion scanner and reports its distinct bound with incomplete required evidence",async()=>{
+ const f=await fixture();const dir=path.join(f.cell,"blob-report");await mkdir(dir);const zip=path.join(dir,"oversized.zip");
+ execFileSync("python3",["-c","import zipfile,sys; z=zipfile.ZipFile(sys.argv[1],'w',compression=zipfile.ZIP_DEFLATED); f=z.open('synthetic.txt','w'); block=b'0'*1048576; [f.write(block) for _ in range(257)]; f.close(); z.close()",zip],{stdio:"pipe",timeout:30000});
+ const evidence=await packageEvidence(f.input);
+ expect(evidence.leaks).toMatchObject([{reason:"zip inspection exceeded its expanded-byte bound"}]);
+ expect(privateAttemptDiagnostic({stage:"seal-original",evidence})).toMatchObject({packagingBudget:{bound:"zip-expanded-bytes",inventoryComplete:true,scanFinished:true,requiredEvidenceComplete:false,qualification:"unqualified"}});
+ expect(evidence.files).not.toContain("blob-report/oversized.zip");
+});
+it("a completed small scan reports exact closed metadata but cannot qualify an outcome without required carriers",async()=>{
+ const f=await fixture();await writeFile(path.join(f.cell,"result.json"),'{"status":"passed"}');await writeFile(path.join(f.cell,"server.log"),"synthetic");
+ const evidence=await packageEvidence(f.input);const d=privateAttemptDiagnostic({stage:"seal-original",result:{status:"passed"},evidence});
+ expect(d).toMatchObject({status:"passed",packagingBudget:{bound:"none",filesExamined:2,bytesExamined:28,inventoryComplete:true,scanFinished:true,requiredEvidenceComplete:false,qualification:"unqualified",byKind:{json:{files:1,bytes:19},log:{files:1,bytes:9}}}});
+ expect(evidence.missing).toContain("final-state.png");
+});
+it("unknown names/strings are discarded and malformed or contradictory counters never escape",()=>{
+ const clean={bound:"none",listedFiles:0,filesExamined:0,bytesExamined:0,inventoryComplete:true,scanFinished:true,requiredEvidenceComplete:null,byKind:Object.fromEntries(["json","log","image","video","zip","html","text","asset"].map(k=>[k,{files:0,bytes:0}]))};
+ for(const mutate of [(b:any)=>{b.bytesExamined=-1;},(b:any)=>{b.listedFiles=4098;},(b:any)=>{b.filesExamined=0.5;},(b:any)=>{b.byKind.json.bytes=1;},(b:any)=>{delete b.byKind.log;},(b:any)=>{b.inventoryComplete=false;b.requiredEvidenceComplete=true;}]){
+  const budget=structuredClone(clean);mutate(budget);expect(privateAttemptDiagnostic({stage:"package-original",evidence:{budget}})).toMatchObject({packagingBudget:null});
+ }
+ const raw={...clean,bound:"private-secret-value",path:"private-secret-value",sessionId:"private-secret-value",byKind:{...clean.byKind,"private-secret-value":{files:1,bytes:1}}};
+ const d=privateAttemptDiagnostic({stage:"package-original",evidence:{budget:raw}});expect(d).toMatchObject({packagingBudget:{bound:"unknown",qualification:"unqualified"}});expect(JSON.stringify(d)).not.toContain("private-secret-value");
+});
+it("sealer failure names its own limit without pretending package counters measure a completed original seal",()=>{
+ const budget={bound:"none",listedFiles:0,filesExamined:0,bytesExamined:0,inventoryComplete:true,scanFinished:true,requiredEvidenceComplete:true,byKind:Object.fromEntries(["json","log","image","video","zip","html","text","asset"].map(k=>[k,{files:0,bytes:0}]))};
+ expect(privateAttemptDiagnostic({stage:"seal-original",evidence:{budget},error:new Error("Complete original evidence exceeds byte allowance")})).toMatchObject({packagingBudget:{scope:"package-standard-files",bound:"original-seal-bytes",qualification:"unqualified"}});
+});
+
+it("entry excess still rejects before file reads and emits only a bounded incomplete counter",async()=>{
+ const f=await fixture();const dir=path.join(f.cell,"snapshots");await mkdir(dir);
+ for(let n=0;n<4097;n+=64)await Promise.all(Array.from({length:Math.min(64,4097-n)},(_,i)=>writeFile(path.join(dir,`synthetic-${n+i}.json`),"{}")));
+ const error=await packageEvidence(f.input).then(()=>null,error=>error);expect(error?.message).toBe("Private cell evidence exceeds entry allowance");
+ expect(privateAttemptDiagnostic({stage:"package-original",error})).toMatchObject({packagingBudget:{bound:"entries",listedFiles:4097,filesExamined:0,bytesExamined:0,inventoryComplete:false,qualification:"unqualified"}});
+});
+it("diagnostic counters cannot rescue a secret-bearing file or admit a symlink",async()=>{
+ const f=await fixture();await writeFile(path.join(f.cell,"server.log"),"synthetic-private-value");
+ const result=await packageEvidence({...f.input,secrets:["synthetic-private-value"]});expect(result.leaks.length).toBeGreaterThan(0);
+ expect(privateAttemptDiagnostic({stage:"package-original",evidence:result})).toMatchObject({packagingBudget:{requiredEvidenceComplete:false,qualification:"unqualified"}});
+ expect(JSON.stringify(privateAttemptDiagnostic({stage:"package-original",evidence:result}))).not.toContain("synthetic-private-value");
+ await rm(path.join(f.cell,"server.log"));await symlink(path.join(f.root,"outside"),path.join(f.cell,"server.log"));
+ const linked=await packageEvidence(f.input);expect(linked.files).not.toContain("server.log");expect(linked.missing).toContain("final-state.png");
+});