diff --git a/tests/runner-e2e/evidence.ts b/tests/runner-e2e/evidence.ts index 4ea6d946e3c..2cc4307e2dc 100644 --- a/tests/runner-e2e/evidence.ts +++ b/tests/runner-e2e/evidence.ts @@ -71,9 +71,32 @@ export function isStandardEvidenceFile(relative: string) { || extension === ".zip"); } +// Diagnostic metadata only: never an admission, completeness or grading receipt. +// inventoryComplete describes the listed standard files, not a sealed original tree. +interface PrivateEvidenceBudget { + bound: string; listedFiles: number; filesExamined: number; bytesExamined: number; + inventoryComplete: boolean; scanFinished: boolean; requiredEvidenceComplete: boolean | null; + byKind: Record; +} +const evidenceBudgetKinds = ["json", "log", "image", "video", "zip", "html", "text", "asset"]; +function evidenceBudgetKind(extension: string): string { + return extension === ".json" ? "json" : extension === ".log" ? "log" : [".png", ".jpg", ".jpeg", ".gif", ".webp"].includes(extension) ? "image" + : [".webm", ".mp4"].includes(extension) ? "video" : extension === ".zip" ? "zip" : extension === ".html" ? "html" + : [".txt", ".md", ".xml"].includes(extension) ? "text" : "asset"; +} +function budgetFailure(message: string, budget: PrivateEvidenceBudget, bound: string) { + budget.bound = bound; + return Object.assign(new Error(message), { evidenceBudget: budget }); +} +function scannerBudgetBound(reason: string): string { + return reason === "zip inspection exceeded its expanded-byte bound" ? "zip-expanded-bytes" + : reason === "zip inspection exceeded its time bound" ? "zip-time" + : reason === "embedded report exceeded its byte bound" ? "html-encoded-bytes" : "none"; +} export interface EvidencePackageResult { files: string[]; leaks: Array<{ file: string; reason: string }>; + budget?: PrivateEvidenceBudget; missing: string[]; } @@ -186,7 +209,10 @@ export async function packageEvidence(input: { const available = await walk(input.privateDir, "", Boolean(input.privateCell)); const standard = available.filter(isStandardEvidencePath); - if (input.privateCell && standard.length > 4096) throw new Error("Private cell evidence exceeds entry allowance"); + const budget: PrivateEvidenceBudget | undefined = input.privateCell ? { bound: "none", listedFiles: Math.min(standard.length, 4097), filesExamined: 0, bytesExamined: 0, + inventoryComplete: false, scanFinished: false, requiredEvidenceComplete: null, + byKind: Object.fromEntries(evidenceBudgetKinds.map(kind => [kind, { files: 0, bytes: 0 }])) } : undefined; + if (input.privateCell && standard.length > 4096) throw budgetFailure("Private cell evidence exceeds entry allowance", budget!, "entries"); const privateSecrets = input.privateCell ? normalizedSecrets(input.secrets).flatMap(value => [value, encodeURIComponent(value), Buffer.from(value).toString("base64"), Buffer.from(value).toString("base64url")]) : []; let privateBytes = 0; for (const relative of standard) { @@ -196,8 +222,12 @@ export async function packageEvidence(input: { if (input.privateCell) { assertSecretFree(relative, privateSecrets, "private cell evidence path"); const metadata = await lstat(source); + if (Number.isSafeInteger(metadata.size) && metadata.size >= 0 && Number.isSafeInteger(budget!.bytesExamined + metadata.size)) { + budget!.filesExamined++; budget!.bytesExamined += metadata.size; + const counter = budget!.byKind[evidenceBudgetKind(extension)]!; counter.files++; counter.bytes += metadata.size; + } if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > 256 * 1024 * 1024 - privateBytes) - throw new Error("Private cell evidence exceeds byte allowance"); + throw budgetFailure("Private cell evidence exceeds byte allowance", budget!, "physical-bytes"); privateBytes += metadata.size; if (!Number.isSafeInteger(privateBytes)) throw new Error("Invalid private cell evidence byte accounting"); const asset = cellTraceAsset(relative, input.privateCell.owner, input.privateCell.layout); @@ -218,7 +248,7 @@ export async function packageEvidence(input: { const raw = await readFile(source, "utf8"); if (extension === ".html") { const issue = await inspectEmbeddedReport(raw, input.secrets); - if (issue) { leaks.push({ file: relative, reason: issue }); continue; } + if (issue) { if (budget && budget.bound === "none") budget.bound = scannerBudgetBound(issue); leaks.push({ file: relative, reason: issue }); continue; } } const parsed = extension === ".json" ? JSON.parse(raw) : null; const leak = @@ -248,6 +278,7 @@ export async function packageEvidence(input: { } else if (extension === ".zip") { const leak = await inspectZip(source, input.secrets); if (leak) { + if (budget && budget.bound === "none") budget.bound = scannerBudgetBound(leak); leaks.push({ file: relative, reason: leak }); continue; } @@ -269,6 +300,7 @@ export async function packageEvidence(input: { } } + if (budget) { budget.inventoryComplete = true; budget.scanFinished = true; } const missing = input.expectPassScreenshot ? [ ...REQUIRED_PASS_FILES.filter((required) => !files.includes(required)), @@ -299,7 +331,8 @@ export async function packageEvidence(input: { "utf8", ); files.push("evidence-manifest.json"); - return { files, leaks, missing }; + if (budget && input.expectPassScreenshot) budget.requiredEvidenceComplete = missing.length === 0 && leaks.length === 0; + return { files, leaks, missing, ...(budget ? { budget } : {}) }; } export async function publishRunnerE2EAttemptEvidence(input: { diff --git a/tests/runner-e2e/private-attempt-diagnostic.ts b/tests/runner-e2e/private-attempt-diagnostic.ts index 22f3a2be4ed..7c72c97937e 100644 --- a/tests/runner-e2e/private-attempt-diagnostic.ts +++ b/tests/runner-e2e/private-attempt-diagnostic.ts @@ -3,6 +3,33 @@ import { lstat, open, readFile, realpath } from "node:fs/promises"; import path from "node:path"; import { PAPERCLIP_PROTOCOL_ACTIONS } from "../../packages/paperclip-runner/src/protocol-actions/index.js"; +// Reject malformed counters wholesale. Unknown input never becomes a diagnostic +// string, path, or authority; even a complete scan remains unqualified evidence. +function closedPackagingBudget(input: unknown, message: string) { + const b = input && typeof input === "object" ? input as Record : {}; + const kinds = ["json", "log", "image", "video", "zip", "html", "text", "asset"]; + const number = (v: unknown) => Number.isSafeInteger(v) && (v as number) >= 0; + if (![b.listedFiles, b.filesExamined, b.bytesExamined].every(number) || b.listedFiles > 4097 || b.filesExamined > 4096 + || b.filesExamined > b.listedFiles || typeof b.inventoryComplete !== "boolean" || typeof b.scanFinished !== "boolean" + || ![true, false, null].includes(b.requiredEvidenceComplete)) return null; + const byKind: Record = {}; + for (const kind of kinds) { + const row = b.byKind?.[kind]; + if (!row || !number(row.files) || !number(row.bytes)) return null; + byKind[kind] = { files: row.files, bytes: row.bytes }; + } + if (Object.values(byKind).reduce((n, r) => n + r.files, 0) !== b.filesExamined + || Object.values(byKind).reduce((n, r) => n + r.bytes, 0) !== b.bytesExamined + || b.inventoryComplete && b.filesExamined !== b.listedFiles || b.scanFinished && !b.inventoryComplete) return null; + if (b.requiredEvidenceComplete === true && (!b.scanFinished || b.bound !== "none")) return null; + const observedBound = ["none", "physical-bytes", "entries", "zip-expanded-bytes", "zip-time", "html-encoded-bytes"].includes(b.bound) ? b.bound : "unknown"; + const bound = message === "Complete original evidence exceeds byte allowance" ? "original-seal-bytes" + : message === "Complete original evidence exceeds entry allowance" ? "original-seal-entries" + : message === "Complete packaged evidence exceeds allowance" ? "final-seal-allowance" : observedBound; + return { scope: "package-standard-files", bound, listedFiles: b.listedFiles, filesExamined: b.filesExamined, bytesExamined: b.bytesExamined, + inventoryComplete: b.inventoryComplete, scanFinished: b.scanFinished, requiredEvidenceComplete: b.requiredEvidenceComplete, + byKind, qualification: "unqualified" }; +} export interface PrivateAttemptDiagnosticInput { stage: unknown; result?: unknown; error?: unknown; observation?: unknown; evidence?: unknown } const record = (value: unknown): Record => value && typeof value === "object" ? value as Record : {}; const member = (value: unknown, choices: string[]) => typeof value === "string" && choices.includes(value) ? value : "unknown"; @@ -71,6 +98,7 @@ export function privateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput) { const evidence = record(input.evidence), leaks = Array.isArray(evidence.leaks) ? evidence.leaks : []; return { schema: "paperclip.private-attempt-diagnostic/v1", + packagingBudget: closedPackagingBudget(record(input.error).evidenceBudget ?? evidence.budget, message), stage: member(input.stage, ["execution", "package-original", "seal-original"]), status: member(result.status, ["passed", "failed", "skipped"]), failureClass: member(result.failureClass, ["candidate_failure", "transient_infrastructure", "permanent_infrastructure", "secret_leak", "cleanup_failure"]), diff --git a/tests/runner-e2e/private-evidence-budget.test.ts b/tests/runner-e2e/private-evidence-budget.test.ts new file mode 100644 index 00000000000..f1cf8fdd2c3 --- /dev/null +++ b/tests/runner-e2e/private-evidence-budget.test.ts @@ -0,0 +1,68 @@ +import { mkdtemp, mkdir, truncate, writeFile, readFile, rm, realpath, symlink } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { execFileSync } from "node:child_process"; +import { afterEach, expect, it } from "vitest"; +import { packageEvidence } from "./evidence.js"; +import { inspectCellReportProducer, privateCellEvidenceOwner } from "./manual-private-report-assets.js"; +import { appendPrivateAttemptDiagnostic, privateAttemptDiagnostic } from "./private-attempt-diagnostic.js"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, {recursive:true,force:true}))); }); +async function fixture() { + const root=await mkdtemp(path.join(await realpath(os.tmpdir()),"closed-budget-"));roots.push(root); + const cell=path.join(root,"capture/cases/activity.agent-feedback");await mkdir(cell,{recursive:true}); + const id="chat-features.runner-opencode.local.activity.agent-feedback"; + const owner=await privateCellEvidenceOwner({root,private:true,runId:"123",attempt:"1",executionIds:[id],cellReportProducer:await inspectCellReportProducer()},id); + return {root,cell,input:{privateDir:cell,uploadDir:path.join(root,"upload"),secrets:[],expectPassScreenshot:true,privateCell:{owner,layout:"original" as const}}}; +} +it("a real sparse oversized source still rejects, but its closed byte witness survives the existing runner diagnostic path",async()=>{ + const f=await fixture();const source=path.join(f.cell,"result.json");await writeFile(source,"");await truncate(source,256*1024*1024+1); + const error=await packageEvidence(f.input).then(()=>null,error=>error);expect(error?.message).toBe("Private cell evidence exceeds byte allowance"); + const d=privateAttemptDiagnostic({stage:"package-original",result:{status:"passed"},error}); + expect(d).toMatchObject({finalization:"byte-bound",packagingBudget:{bound:"physical-bytes",inventoryComplete:false,scanFinished:false,requiredEvidenceComplete:null,qualification:"unqualified",filesExamined:1,bytesExamined:268435457,byKind:{json:{files:1,bytes:268435457}}}}); + const commands=path.join(f.root,"_runner_file_commands");await mkdir(commands);const envFile=path.join(commands,"set_env_budget");await writeFile(envFile,""); + await appendPrivateAttemptDiagnostic({stage:"package-original",result:{status:"passed"},error},{GITHUB_ACTIONS:"true",RUNNER_TEMP:f.root,GITHUB_ENV:envFile}); + const wire=await readFile(envFile,"utf8");expect(wire).toContain('"bound":"physical-bytes"');expect(wire).not.toContain(source);expect(wire).not.toContain("closed-budget-");expect(wire.trim().split("\n")).toHaveLength(1); +}); + +it("retains the real ZIP expansion scanner and reports its distinct bound with incomplete required evidence",async()=>{ + const f=await fixture();const dir=path.join(f.cell,"blob-report");await mkdir(dir);const zip=path.join(dir,"oversized.zip"); + execFileSync("python3",["-c","import zipfile,sys; z=zipfile.ZipFile(sys.argv[1],'w',compression=zipfile.ZIP_DEFLATED); f=z.open('synthetic.txt','w'); block=b'0'*1048576; [f.write(block) for _ in range(257)]; f.close(); z.close()",zip],{stdio:"pipe",timeout:30000}); + const evidence=await packageEvidence(f.input); + expect(evidence.leaks).toMatchObject([{reason:"zip inspection exceeded its expanded-byte bound"}]); + expect(privateAttemptDiagnostic({stage:"seal-original",evidence})).toMatchObject({packagingBudget:{bound:"zip-expanded-bytes",inventoryComplete:true,scanFinished:true,requiredEvidenceComplete:false,qualification:"unqualified"}}); + expect(evidence.files).not.toContain("blob-report/oversized.zip"); +}); +it("a completed small scan reports exact closed metadata but cannot qualify an outcome without required carriers",async()=>{ + const f=await fixture();await writeFile(path.join(f.cell,"result.json"),'{"status":"passed"}');await writeFile(path.join(f.cell,"server.log"),"synthetic"); + const evidence=await packageEvidence(f.input);const d=privateAttemptDiagnostic({stage:"seal-original",result:{status:"passed"},evidence}); + expect(d).toMatchObject({status:"passed",packagingBudget:{bound:"none",filesExamined:2,bytesExamined:28,inventoryComplete:true,scanFinished:true,requiredEvidenceComplete:false,qualification:"unqualified",byKind:{json:{files:1,bytes:19},log:{files:1,bytes:9}}}}); + expect(evidence.missing).toContain("final-state.png"); +}); +it("unknown names/strings are discarded and malformed or contradictory counters never escape",()=>{ + const clean={bound:"none",listedFiles:0,filesExamined:0,bytesExamined:0,inventoryComplete:true,scanFinished:true,requiredEvidenceComplete:null,byKind:Object.fromEntries(["json","log","image","video","zip","html","text","asset"].map(k=>[k,{files:0,bytes:0}]))}; + for(const mutate of [(b:any)=>{b.bytesExamined=-1;},(b:any)=>{b.listedFiles=4098;},(b:any)=>{b.filesExamined=0.5;},(b:any)=>{b.byKind.json.bytes=1;},(b:any)=>{delete b.byKind.log;},(b:any)=>{b.inventoryComplete=false;b.requiredEvidenceComplete=true;}]){ + const budget=structuredClone(clean);mutate(budget);expect(privateAttemptDiagnostic({stage:"package-original",evidence:{budget}})).toMatchObject({packagingBudget:null}); + } + const raw={...clean,bound:"private-secret-value",path:"private-secret-value",sessionId:"private-secret-value",byKind:{...clean.byKind,"private-secret-value":{files:1,bytes:1}}}; + const d=privateAttemptDiagnostic({stage:"package-original",evidence:{budget:raw}});expect(d).toMatchObject({packagingBudget:{bound:"unknown",qualification:"unqualified"}});expect(JSON.stringify(d)).not.toContain("private-secret-value"); +}); +it("sealer failure names its own limit without pretending package counters measure a completed original seal",()=>{ + const budget={bound:"none",listedFiles:0,filesExamined:0,bytesExamined:0,inventoryComplete:true,scanFinished:true,requiredEvidenceComplete:true,byKind:Object.fromEntries(["json","log","image","video","zip","html","text","asset"].map(k=>[k,{files:0,bytes:0}]))}; + expect(privateAttemptDiagnostic({stage:"seal-original",evidence:{budget},error:new Error("Complete original evidence exceeds byte allowance")})).toMatchObject({packagingBudget:{scope:"package-standard-files",bound:"original-seal-bytes",qualification:"unqualified"}}); +}); + +it("entry excess still rejects before file reads and emits only a bounded incomplete counter",async()=>{ + const f=await fixture();const dir=path.join(f.cell,"snapshots");await mkdir(dir); + for(let n=0;n<4097;n+=64)await Promise.all(Array.from({length:Math.min(64,4097-n)},(_,i)=>writeFile(path.join(dir,`synthetic-${n+i}.json`),"{}"))); + const error=await packageEvidence(f.input).then(()=>null,error=>error);expect(error?.message).toBe("Private cell evidence exceeds entry allowance"); + expect(privateAttemptDiagnostic({stage:"package-original",error})).toMatchObject({packagingBudget:{bound:"entries",listedFiles:4097,filesExamined:0,bytesExamined:0,inventoryComplete:false,qualification:"unqualified"}}); +}); +it("diagnostic counters cannot rescue a secret-bearing file or admit a symlink",async()=>{ + const f=await fixture();await writeFile(path.join(f.cell,"server.log"),"synthetic-private-value"); + const result=await packageEvidence({...f.input,secrets:["synthetic-private-value"]});expect(result.leaks.length).toBeGreaterThan(0); + expect(privateAttemptDiagnostic({stage:"package-original",evidence:result})).toMatchObject({packagingBudget:{requiredEvidenceComplete:false,qualification:"unqualified"}}); + expect(JSON.stringify(privateAttemptDiagnostic({stage:"package-original",evidence:result}))).not.toContain("synthetic-private-value"); + await rm(path.join(f.cell,"server.log"));await symlink(path.join(f.root,"outside"),path.join(f.cell,"server.log")); + const linked=await packageEvidence(f.input);expect(linked.files).not.toContain("server.log");expect(linked.missing).toContain("final-state.png"); +});