diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 39a8bf7a211..dba1467e211 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -871,7 +871,7 @@ async function runAttempt(input: { `attempt-${attempt}`, ); await mkdir(casePrivateDir, { recursive: true }); - await copySharedEvidence(privateDir, casePrivateDir, temporaryRoot, credentials); + await copySharedEvidence(privateDir, casePrivateDir, temporaryRoot, credentials, manualPrivateHandoff ? { executionId: candidate.id, context: manualPrivateHandoff } : undefined); if (reporterPrivacy.length) { const original = { status: result.status, failureClass: result.failureClass, error: result.error }; const snapshots = path.join(casePrivateDir, "snapshots"); await mkdir(snapshots, { recursive: true }); diff --git a/tests/runner-e2e/shared-evidence.ts b/tests/runner-e2e/shared-evidence.ts index 6c16051025b..85738682722 100644 --- a/tests/runner-e2e/shared-evidence.ts +++ b/tests/runner-e2e/shared-evidence.ts @@ -1,3 +1,4 @@ +import type { SourceSdkAuditCapture } from "./source-sdk-evidence.js"; import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { createHash } from "node:crypto"; @@ -5,7 +6,7 @@ import { runnerE2EServerControlPaths } from "./harness-env.js"; import { redactText, sanitizeJson } from "./redaction.js"; export async function copySharedEvidence(privateRoot: string, casePrivateDir: string, - temporaryRoot: string, secrets: readonly string[]) { + temporaryRoot: string, secrets: readonly string[], sdkAudit?: SourceSdkAuditCapture) { for (const relative of [ "server.log", "playwright.log", @@ -22,6 +23,7 @@ export async function copySharedEvidence(privateRoot: string, casePrivateDir: st if (error.code !== "ENOENT") throw error; }); } + if (sdkAudit) await (await import("./source-sdk-evidence.js")).copyPreparedSourceSdkAudit(temporaryRoot, casePrivateDir, secrets, sdkAudit); // The supervisor records each phase before it can fail. Copy its receipts // independently of the successful-restart acknowledgement before cleanup. const { controlDirectory } = runnerE2EServerControlPaths(temporaryRoot); diff --git a/tests/runner-e2e/source-sdk-audit-capture.test.ts b/tests/runner-e2e/source-sdk-audit-capture.test.ts new file mode 100644 index 00000000000..a80d8d17ecb --- /dev/null +++ b/tests/runner-e2e/source-sdk-audit-capture.test.ts @@ -0,0 +1,55 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, writeFile, readFile, realpath, rm, symlink, stat } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { afterEach, expect, it } from "vitest"; +import { copyPreparedSourceSdkAudit, retainPreparedSourceSdk } from "./source-sdk-evidence.js"; +import { copySharedEvidence } from "./shared-evidence.js"; +import { inspectCellReportProducer } from "./manual-private-report-assets.js"; +import { stagePrivateDiagnostics } from "./private-diagnostics.js"; +const roots:string[]=[],id="chat-features.runner-opencode.local.plugins.configure-plugin",scenario="plugins.configure-plugin"; +afterEach(async()=>{await Promise.all(roots.splice(0).map(root=>rm(root,{recursive:true,force:true})));}); +const hash=(v:string|Buffer)=>createHash("sha256").update(v).digest("hex"); +async function fixture(){ + const root=await realpath(await mkdtemp(path.join(os.tmpdir(),"sdk-audit-capture-")));roots.push(root); + const temp=path.join(root,"attempt"),repo=path.join(root,"repo"),handoff=path.join(root,"handoff"),capture=path.join(handoff,"capture"),cell=path.join(capture,"cases",scenario); + await mkdir(temp,{mode:0o700});await mkdir(handoff,{mode:0o700});await mkdir(cell,{recursive:true,mode:0o700}); + await mkdir(path.join(temp,"selected-private-inputs"),{mode:0o700}); + const secret="synthetic-private-sdk-canary-value";stagePrivateDiagnostics(temp,[id],[secret]); + for(const directory of ["packages/shared/dist","packages/plugins/sdk/dist"]){ + const dir=path.join(repo,directory);await mkdir(dir,{recursive:true});const body=Buffer.from(secret); + await writeFile(path.join(dir,"index.js"),body);const outputs=createHash("sha256").update(JSON.stringify(["index.js",body.length])).update(body).digest("hex"); + await writeFile(path.join(dir,".paperclip-build-complete"),JSON.stringify({sources:"a".repeat(64),outputs})); + } + const context={root:handoff,private:true,runId:"123",attempt:"1",executionIds:[id],cellReportProducer:await inspectCellReportProducer()}; + const source=path.join(temp,"artifacts-private/cases",scenario,"snapshots/source-sdk-privacy-observations.json"),destination=path.join(cell,"snapshots/source-sdk-privacy-observations.json"); + const reject=()=>retainPreparedSourceSdk({repositoryRoot:repo,temporaryRoot:temp,executionIds:[id],sourceRevision:"a".repeat(40),installed:false}); + const copy=()=>copySharedEvidence(capture,cell,temp,[secret],{executionId:id,context}); + return {root,temp,repo,handoff,capture,cell,context,source,destination,secret,reject,copy}; +} +it("actual SDK privacy rejection keeps its existing typed audit in the external private bootstrap-failure capture",async()=>{ + const f=await fixture();await expect(f.reject()).rejects.toThrow("Source SDK evidence privacy check failed; bytes withheld"); + const bytes=await readFile(f.source);expect(JSON.parse(bytes.toString())).toMatchObject({complete:false,audit:{observations:expect.arrayContaining([expect.objectContaining({classification:"private_exact"})])}}); + await f.copy();expect(await readFile(f.destination)).toEqual(bytes);expect((await stat(f.destination)).mode&0o777).toBe(0o600); + expect((await readFile(f.destination,"utf8"))).not.toContain(f.secret);await expect(stat(path.join(f.cell,"source-sdk-build"))).rejects.toThrow(); +}); + +it.each(["symlink file","foreign capture","wrong execution","wrong producer","private canary","unknown shape","oversize audit"])("rejects %s before any external audit write",async fault=>{ + const f=await fixture();await expect(f.reject()).rejects.toThrow("Source SDK evidence privacy check failed"); + let cell=f.cell,input={executionId:id,context:f.context}; + if(fault==="symlink file"){const outside=path.join(f.root,"outside.json");await writeFile(outside,await readFile(f.source));await rm(f.source);await symlink(outside,f.source);} + if(fault==="foreign capture"){cell=path.join(f.root,"foreign");await mkdir(cell);} + if(fault==="wrong execution")input={...input,context:{...f.context,executionIds:["chat-features.runner-opencode.local.plugins.disable-plugin"]}}; + if(fault==="wrong producer")input={...input,context:{...f.context,cellReportProducer:{...f.context.cellReportProducer,staticManifestSha256:"0".repeat(64)}}}; + if(fault==="private canary"){const audit=JSON.parse(await readFile(f.source,"utf8"));audit.private=f.secret;await writeFile(f.source,JSON.stringify(audit));} + if(fault==="unknown shape"){const audit=JSON.parse(await readFile(f.source,"utf8"));audit.path="private-unapproved-path";await writeFile(f.source,JSON.stringify(audit));} + if(fault==="oversize audit")await writeFile(f.source,Buffer.alloc(1024*1024+1)); + await expect(copyPreparedSourceSdkAudit(f.temp,cell,[f.secret],input)).rejects.toThrow(); + await expect(stat(f.destination)).rejects.toThrow(); +}); +it("keeps the captured audit immutable and leaves non-SDK/missing-audit paths unchanged",async()=>{ + const f=await fixture();await f.copy();await expect(stat(f.destination)).rejects.toThrow(); + await expect(copyPreparedSourceSdkAudit(f.temp,f.cell,[],{executionId:"chat-features.runner-opencode.local.tasks.ambiguous-task",context:f.context})).resolves.toBeUndefined(); + await expect(f.reject()).rejects.toThrow("Source SDK evidence privacy check failed");await f.copy();const original=await readFile(f.destination); + await expect(f.copy()).rejects.toThrow();expect(await readFile(f.destination)).toEqual(original); +}); diff --git a/tests/runner-e2e/source-sdk-evidence.ts b/tests/runner-e2e/source-sdk-evidence.ts index d21e9bb676e..7a33a224cc8 100644 --- a/tests/runner-e2e/source-sdk-evidence.ts +++ b/tests/runner-e2e/source-sdk-evidence.ts @@ -1,6 +1,8 @@ import { createSdkArtifactScanner } from "./source-sdk-public-type.js"; +import { constants } from "node:fs"; +import { assertPrivateCellDirectories, privateCellEvidenceOwner } from "./manual-private-report-assets.js"; import { createHash } from "node:crypto"; -import { chmod, lstat, mkdir, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; +import { chmod, lstat, mkdir, open, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { diagnosticSecrets, readPrivateDiagnosticSecrets } from "./private-diagnostics.js"; import { findSecretLeak } from "./redaction.js"; @@ -131,3 +133,48 @@ export async function copyRetainedSourceSdk(root: string, executionId: string, s if (hash(await readFile(filename)) !== file.sha256) throw new Error("Source SDK published bytes changed"); } } + +export interface SourceSdkAuditCapture { executionId: string; context: Parameters[0] } +function closedSdkAudit(value: unknown): boolean { + const object = (v: any, keys: string[]) => !!v && typeof v === "object" && !Array.isArray(v) && Object.keys(v).sort().join() === [...keys].sort().join(); + const v = value as any, a = v?.audit; + if (!object(v, ["schema", "complete", "audit"]) || v.schema !== "paperclip.source-sdk-privacy-observation/v1" || typeof v.complete !== "boolean" + || !object(a, ["schema", "suppressedFindings", "bounded", "publicSource", "observations"]) || a.schema !== "paperclip.source-sdk-scan/v1" || a.suppressedFindings !== 0 + || typeof a.bounded !== "boolean" || !Array.isArray(a.observations) || a.observations.length > 4096) return false; + if (a.publicSource !== null) { + const hashes = ["manifestSha256", "lockSha256", "compilerPackageSha256", "parserPackageSha256", "parserEntrySha256", "zodPackageSha256", "zodEntrySha256", "zodTypeGraphSha256"]; + if (!object(a.publicSource, ["inputs", ...hashes]) || !Number.isSafeInteger(a.publicSource.inputs) || a.publicSource.inputs < 1 || a.publicSource.inputs > 20000 || !hashes.every(k => hex(a.publicSource[k]))) return false; + } + return a.observations.every((r: any) => object(r, ["pathSha256", "fileSha256", "detector", "start", "end", "classification"]) + && hex(r.pathSha256) && hex(r.fileSha256) && Number.isInteger(r.detector) && r.detector >= -1 && r.detector <= 3 + && Number.isSafeInteger(r.start) && Number.isSafeInteger(r.end) && r.start >= 0 && r.end >= r.start && r.end <= 8 * 1024 * 1024 + && (r.detector === -1 ? r.classification === "private_exact" : r.detector === 3 ? ["unresolved_context", "public_schema_type"].includes(r.classification) : r.classification === "direct_token")); +} +/** Copy only the already-produced typed audit, never rejected SDK output. The + * private producer owner binds external capture even when server setup failed. */ +export async function copyPreparedSourceSdkAudit(temporaryRoot: string, casePrivateDir: string, secrets: readonly string[], input: SourceSdkAuditCapture) { + if (!selected([input.executionId])) return; + const owner = await privateCellEvidenceOwner(input.context, input.executionId); + const scenario = input.executionId.slice("chat-features.runner-opencode.local.".length); + if (path.resolve(casePrivateDir) !== path.join(owner.root, "capture", "cases", scenario)) throw new Error("SDK audit capture does not match its exact private cell"); + await assertPrivateCellDirectories(owner, casePrivateDir); + const root = await realpath(temporaryRoot), rootInfo = await lstat(temporaryRoot); + if (root !== path.resolve(temporaryRoot) || !rootInfo.isDirectory() || rootInfo.isSymbolicLink() || rootInfo.mode & 0o077 + || typeof process.getuid === "function" && rootInfo.uid !== process.getuid()) throw new Error("SDK audit source custody invalid"); + const source = path.join(root, "artifacts-private", "cases", scenario, "snapshots", "source-sdk-privacy-observations.json"); + const info = await lstat(source).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return; throw error; }); + if (!info) return; + if (!info.isFile() || info.isSymbolicLink() || info.size > 1024 * 1024 || await realpath(source) !== source) throw new Error("SDK audit source custody invalid"); + const handle = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW); + let bytes: Buffer; + try { + const before = await handle.stat(); bytes = await handle.readFile(); const after = await handle.stat(); + if (!before.isFile() || before.ino !== info.ino || before.dev !== info.dev || before.size !== info.size || bytes.length !== info.size + || before.ino !== after.ino || before.dev !== after.dev || before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs || await realpath(source) !== source) throw new Error("SDK audit changed during capture"); + } finally { await handle.close(); } + checkSecret(bytes, secrets); + if (!closedSdkAudit(JSON.parse(bytes.toString("utf8")))) throw new Error("SDK audit has an unsupported observation shape"); + const destination = path.join(casePrivateDir, "snapshots"); await mkdir(destination, { recursive: true, mode: 0o700 }); + await assertPrivateCellDirectories(owner, destination); + await writeFile(path.join(destination, "source-sdk-privacy-observations.json"), bytes, { flag: "wx", mode: 0o600 }); +}