fix(evals): retain SDK scan audit after bootstrap rejection

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/launch.ts

@@ -871,7 +871,7 @@ async function runAttempt(input: {
         `attempt-${attempt}`,
       );
       await mkdir(casePrivateDir, { recursive: true });
-      await copySharedEvidence(privateDir, casePrivateDir, temporaryRoot, credentials);
+      await copySharedEvidence(privateDir, casePrivateDir, temporaryRoot, credentials, manualPrivateHandoff ? { executionId: candidate.id, context: manualPrivateHandoff } : undefined);
       if (reporterPrivacy.length) {
         const original = { status: result.status, failureClass: result.failureClass, error: result.error };
         const snapshots = path.join(casePrivateDir, "snapshots"); await mkdir(snapshots, { recursive: true });

tests/runner-e2e/shared-evidence.ts

@@ -1,3 +1,4 @@
+import type { SourceSdkAuditCapture } from "./source-sdk-evidence.js";
 import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { createHash } from "node:crypto";
@@ -5,7 +6,7 @@ import { runnerE2EServerControlPaths } from "./harness-env.js";
 import { redactText, sanitizeJson } from "./redaction.js";
 
 export async function copySharedEvidence(privateRoot: string, casePrivateDir: string,
-  temporaryRoot: string, secrets: readonly string[]) {
+  temporaryRoot: string, secrets: readonly string[], sdkAudit?: SourceSdkAuditCapture) {
   for (const relative of [
     "server.log",
     "playwright.log",
@@ -22,6 +23,7 @@ export async function copySharedEvidence(privateRoot: string, casePrivateDir: st
       if (error.code !== "ENOENT") throw error;
     });
   }
+  if (sdkAudit) await (await import("./source-sdk-evidence.js")).copyPreparedSourceSdkAudit(temporaryRoot, casePrivateDir, secrets, sdkAudit);
   // The supervisor records each phase before it can fail. Copy its receipts
   // independently of the successful-restart acknowledgement before cleanup.
   const { controlDirectory } = runnerE2EServerControlPaths(temporaryRoot);

tests/runner-e2e/source-sdk-audit-capture.test.ts

@@ -0,0 +1,55 @@
+import { createHash } from "node:crypto";
+import { mkdtemp, mkdir, writeFile, readFile, realpath, rm, symlink, stat } from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import { afterEach, expect, it } from "vitest";
+import { copyPreparedSourceSdkAudit, retainPreparedSourceSdk } from "./source-sdk-evidence.js";
+import { copySharedEvidence } from "./shared-evidence.js";
+import { inspectCellReportProducer } from "./manual-private-report-assets.js";
+import { stagePrivateDiagnostics } from "./private-diagnostics.js";
+const roots:string[]=[],id="chat-features.runner-opencode.local.plugins.configure-plugin",scenario="plugins.configure-plugin";
+afterEach(async()=>{await Promise.all(roots.splice(0).map(root=>rm(root,{recursive:true,force:true})));});
+const hash=(v:string|Buffer)=>createHash("sha256").update(v).digest("hex");
+async function fixture(){
+ const root=await realpath(await mkdtemp(path.join(os.tmpdir(),"sdk-audit-capture-")));roots.push(root);
+ const temp=path.join(root,"attempt"),repo=path.join(root,"repo"),handoff=path.join(root,"handoff"),capture=path.join(handoff,"capture"),cell=path.join(capture,"cases",scenario);
+ await mkdir(temp,{mode:0o700});await mkdir(handoff,{mode:0o700});await mkdir(cell,{recursive:true,mode:0o700});
+ await mkdir(path.join(temp,"selected-private-inputs"),{mode:0o700});
+ const secret="synthetic-private-sdk-canary-value";stagePrivateDiagnostics(temp,[id],[secret]);
+ for(const directory of ["packages/shared/dist","packages/plugins/sdk/dist"]){
+  const dir=path.join(repo,directory);await mkdir(dir,{recursive:true});const body=Buffer.from(secret);
+  await writeFile(path.join(dir,"index.js"),body);const outputs=createHash("sha256").update(JSON.stringify(["index.js",body.length])).update(body).digest("hex");
+  await writeFile(path.join(dir,".paperclip-build-complete"),JSON.stringify({sources:"a".repeat(64),outputs}));
+ }
+ const context={root:handoff,private:true,runId:"123",attempt:"1",executionIds:[id],cellReportProducer:await inspectCellReportProducer()};
+ const source=path.join(temp,"artifacts-private/cases",scenario,"snapshots/source-sdk-privacy-observations.json"),destination=path.join(cell,"snapshots/source-sdk-privacy-observations.json");
+ const reject=()=>retainPreparedSourceSdk({repositoryRoot:repo,temporaryRoot:temp,executionIds:[id],sourceRevision:"a".repeat(40),installed:false});
+ const copy=()=>copySharedEvidence(capture,cell,temp,[secret],{executionId:id,context});
+ return {root,temp,repo,handoff,capture,cell,context,source,destination,secret,reject,copy};
+}
+it("actual SDK privacy rejection keeps its existing typed audit in the external private bootstrap-failure capture",async()=>{
+ const f=await fixture();await expect(f.reject()).rejects.toThrow("Source SDK evidence privacy check failed; bytes withheld");
+ const bytes=await readFile(f.source);expect(JSON.parse(bytes.toString())).toMatchObject({complete:false,audit:{observations:expect.arrayContaining([expect.objectContaining({classification:"private_exact"})])}});
+ await f.copy();expect(await readFile(f.destination)).toEqual(bytes);expect((await stat(f.destination)).mode&0o777).toBe(0o600);
+ expect((await readFile(f.destination,"utf8"))).not.toContain(f.secret);await expect(stat(path.join(f.cell,"source-sdk-build"))).rejects.toThrow();
+});
+
+it.each(["symlink file","foreign capture","wrong execution","wrong producer","private canary","unknown shape","oversize audit"])("rejects %s before any external audit write",async fault=>{
+ const f=await fixture();await expect(f.reject()).rejects.toThrow("Source SDK evidence privacy check failed");
+ let cell=f.cell,input={executionId:id,context:f.context};
+ if(fault==="symlink file"){const outside=path.join(f.root,"outside.json");await writeFile(outside,await readFile(f.source));await rm(f.source);await symlink(outside,f.source);}
+ if(fault==="foreign capture"){cell=path.join(f.root,"foreign");await mkdir(cell);}
+ if(fault==="wrong execution")input={...input,context:{...f.context,executionIds:["chat-features.runner-opencode.local.plugins.disable-plugin"]}};
+ if(fault==="wrong producer")input={...input,context:{...f.context,cellReportProducer:{...f.context.cellReportProducer,staticManifestSha256:"0".repeat(64)}}};
+ if(fault==="private canary"){const audit=JSON.parse(await readFile(f.source,"utf8"));audit.private=f.secret;await writeFile(f.source,JSON.stringify(audit));}
+ if(fault==="unknown shape"){const audit=JSON.parse(await readFile(f.source,"utf8"));audit.path="private-unapproved-path";await writeFile(f.source,JSON.stringify(audit));}
+ if(fault==="oversize audit")await writeFile(f.source,Buffer.alloc(1024*1024+1));
+ await expect(copyPreparedSourceSdkAudit(f.temp,cell,[f.secret],input)).rejects.toThrow();
+ await expect(stat(f.destination)).rejects.toThrow();
+});
+it("keeps the captured audit immutable and leaves non-SDK/missing-audit paths unchanged",async()=>{
+ const f=await fixture();await f.copy();await expect(stat(f.destination)).rejects.toThrow();
+ await expect(copyPreparedSourceSdkAudit(f.temp,f.cell,[],{executionId:"chat-features.runner-opencode.local.tasks.ambiguous-task",context:f.context})).resolves.toBeUndefined();
+ await expect(f.reject()).rejects.toThrow("Source SDK evidence privacy check failed");await f.copy();const original=await readFile(f.destination);
+ await expect(f.copy()).rejects.toThrow();expect(await readFile(f.destination)).toEqual(original);
+});

tests/runner-e2e/source-sdk-evidence.ts

@@ -1,6 +1,8 @@
 import { createSdkArtifactScanner } from "./source-sdk-public-type.js";
+import { constants } from "node:fs";
+import { assertPrivateCellDirectories, privateCellEvidenceOwner } from "./manual-private-report-assets.js";
 import { createHash } from "node:crypto";
-import { chmod, lstat, mkdir, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises";
+import { chmod, lstat, mkdir, open, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { diagnosticSecrets, readPrivateDiagnosticSecrets } from "./private-diagnostics.js";
 import { findSecretLeak } from "./redaction.js";
@@ -131,3 +133,48 @@ export async function copyRetainedSourceSdk(root: string, executionId: string, s
     if (hash(await readFile(filename)) !== file.sha256) throw new Error("Source SDK published bytes changed");
   }
 }
+
+export interface SourceSdkAuditCapture { executionId: string; context: Parameters<typeof privateCellEvidenceOwner>[0] }
+function closedSdkAudit(value: unknown): boolean {
+  const object = (v: any, keys: string[]) => !!v && typeof v === "object" && !Array.isArray(v) && Object.keys(v).sort().join() === [...keys].sort().join();
+  const v = value as any, a = v?.audit;
+  if (!object(v, ["schema", "complete", "audit"]) || v.schema !== "paperclip.source-sdk-privacy-observation/v1" || typeof v.complete !== "boolean"
+    || !object(a, ["schema", "suppressedFindings", "bounded", "publicSource", "observations"]) || a.schema !== "paperclip.source-sdk-scan/v1" || a.suppressedFindings !== 0
+    || typeof a.bounded !== "boolean" || !Array.isArray(a.observations) || a.observations.length > 4096) return false;
+  if (a.publicSource !== null) {
+    const hashes = ["manifestSha256", "lockSha256", "compilerPackageSha256", "parserPackageSha256", "parserEntrySha256", "zodPackageSha256", "zodEntrySha256", "zodTypeGraphSha256"];
+    if (!object(a.publicSource, ["inputs", ...hashes]) || !Number.isSafeInteger(a.publicSource.inputs) || a.publicSource.inputs < 1 || a.publicSource.inputs > 20000 || !hashes.every(k => hex(a.publicSource[k]))) return false;
+  }
+  return a.observations.every((r: any) => object(r, ["pathSha256", "fileSha256", "detector", "start", "end", "classification"])
+    && hex(r.pathSha256) && hex(r.fileSha256) && Number.isInteger(r.detector) && r.detector >= -1 && r.detector <= 3
+    && Number.isSafeInteger(r.start) && Number.isSafeInteger(r.end) && r.start >= 0 && r.end >= r.start && r.end <= 8 * 1024 * 1024
+    && (r.detector === -1 ? r.classification === "private_exact" : r.detector === 3 ? ["unresolved_context", "public_schema_type"].includes(r.classification) : r.classification === "direct_token"));
+}
+/** Copy only the already-produced typed audit, never rejected SDK output. The
+ * private producer owner binds external capture even when server setup failed. */
+export async function copyPreparedSourceSdkAudit(temporaryRoot: string, casePrivateDir: string, secrets: readonly string[], input: SourceSdkAuditCapture) {
+  if (!selected([input.executionId])) return;
+  const owner = await privateCellEvidenceOwner(input.context, input.executionId);
+  const scenario = input.executionId.slice("chat-features.runner-opencode.local.".length);
+  if (path.resolve(casePrivateDir) !== path.join(owner.root, "capture", "cases", scenario)) throw new Error("SDK audit capture does not match its exact private cell");
+  await assertPrivateCellDirectories(owner, casePrivateDir);
+  const root = await realpath(temporaryRoot), rootInfo = await lstat(temporaryRoot);
+  if (root !== path.resolve(temporaryRoot) || !rootInfo.isDirectory() || rootInfo.isSymbolicLink() || rootInfo.mode & 0o077
+    || typeof process.getuid === "function" && rootInfo.uid !== process.getuid()) throw new Error("SDK audit source custody invalid");
+  const source = path.join(root, "artifacts-private", "cases", scenario, "snapshots", "source-sdk-privacy-observations.json");
+  const info = await lstat(source).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return; throw error; });
+  if (!info) return;
+  if (!info.isFile() || info.isSymbolicLink() || info.size > 1024 * 1024 || await realpath(source) !== source) throw new Error("SDK audit source custody invalid");
+  const handle = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW);
+  let bytes: Buffer;
+  try {
+    const before = await handle.stat(); bytes = await handle.readFile(); const after = await handle.stat();
+    if (!before.isFile() || before.ino !== info.ino || before.dev !== info.dev || before.size !== info.size || bytes.length !== info.size
+      || before.ino !== after.ino || before.dev !== after.dev || before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs || await realpath(source) !== source) throw new Error("SDK audit changed during capture");
+  } finally { await handle.close(); }
+  checkSecret(bytes, secrets);
+  if (!closedSdkAudit(JSON.parse(bytes.toString("utf8")))) throw new Error("SDK audit has an unsupported observation shape");
+  const destination = path.join(casePrivateDir, "snapshots"); await mkdir(destination, { recursive: true, mode: 0o700 });
+  await assertPrivateCellDirectories(owner, destination);
+  await writeFile(path.join(destination, "source-sdk-privacy-observations.json"), bytes, { flag: "wx", mode: 0o600 });
+}