From 712dc98cf52a26fafd1c894e40c081da991c4fa7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 2 Oct 2026 12:31:35 -0500 Subject: [PATCH] fix(evals): bind protocol evidence to the built daemon Co-Authored-By: Paperclip --- .../src/live/runnerd-codex-transport.test.ts | 4 +++- tests/runner-e2e/STOCK-HARNESS.md | 3 +++ tests/runner-e2e/stock-harness-checks.mjs | 10 +++++++++- tests/runner-e2e/stock-harness-checks.test.mjs | 5 +++-- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 8ab547d221..4a38700939 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -4154,7 +4154,9 @@ it("captures exact provider frames and correlates Rust and TypeScript interpreta ); const tracePath = join(traceDirectory, "trace.ndjson"); const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), + // Qualification builds a debug daemon for this exact source. Its selected + // binary must win over any separately staged product/runtime artifact. + runnerBinary: process.env.PAPERCLIP_STOCK_PREFLIGHT_RUNNERD ?? defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, codexArgs: fakeCodexArgs(traceDirectory, "--structured-activity"), stateDirectory: join(traceDirectory, "state"), diff --git a/tests/runner-e2e/STOCK-HARNESS.md b/tests/runner-e2e/STOCK-HARNESS.md index ba2b8fe2f5..70ecfb1827 100644 --- a/tests/runner-e2e/STOCK-HARNESS.md +++ b/tests/runner-e2e/STOCK-HARNESS.md @@ -148,6 +148,9 @@ prompt/oracle checks and the Rust additive test. It stopped before providers: the real daemon-frame test lacked the cold `paperclip-runnerd` binary. Setup now builds that daemon from the locked Rust source before TypeScript gates, retains `runnerd-build.txt`, and requires both setup exits in the admission receipt. +The required daemon-frame test selects that built debug binary explicitly, +without replacing staged product binaries. The receipt records its SHA-256; +verification rejects a changed binary before provider admission. Dispatch the trusted workflow from `master`, with `target_branch` naming the same-repository candidate and an exact cell selector first. The workflow resolves diff --git a/tests/runner-e2e/stock-harness-checks.mjs b/tests/runner-e2e/stock-harness-checks.mjs index e8e88a1d00..b818e4673b 100644 --- a/tests/runner-e2e/stock-harness-checks.mjs +++ b/tests/runner-e2e/stock-harness-checks.mjs @@ -90,6 +90,7 @@ export function assertPreflightReceipt(report, current) { if (report?.schema !== "paperclip.stock-harness-preflight.v3" || report.passed !== true || report.setup?.passed !== true || report.setup?.exitCode !== 0 || report.setup?.sdkExitCode !== 0 || report.setup?.runnerdExitCode !== 0 || + report.setup?.runnerdSha256 !== current.runnerdSha256 || report.providerCalls !== 0 || report.sourceSha !== current.sha || report.sourceFingerprint !== current.fingerprint || report.sourceErrors?.length !== 0 || !Array.isArray(report.gates) || report.gates.length !== expected.length || @@ -113,6 +114,8 @@ export function main(args = process.argv.slice(2)) { if (git.status !== 0 || source.sourceErrors.length) throw new Error("Cannot verify stock harness source provenance."); const report = assertPreflightReceipt(JSON.parse(readFileSync(verify, "utf8")), { sha: git.stdout.trim(), fingerprint: source.fingerprint, + runnerdSha256: createHash("sha256").update(readFileSync(join(root, + "packages/paperclip-runner/runner/target/debug", `paperclip-runnerd${process.platform === "win32" ? ".exe" : ""}`))).digest("hex"), }); const output = resolve(verify, ".."); for (const gate of stockHarnessGates) { @@ -158,6 +161,9 @@ export function main(args = process.argv.slice(2)) { process.exitCode = 1; return; } + const runnerdBinary = join(root, "packages/paperclip-runner/runner/target/debug", + `paperclip-runnerd${process.platform === "win32" ? ".exe" : ""}`); + setup.runnerdSha256 = createHash("sha256").update(readFileSync(runnerdBinary)).digest("hex"); const results = []; for (const gate of stockHarnessGates) { console.log(`Checking ${gate.id}: ${gate.name}`); @@ -166,7 +172,9 @@ export function main(args = process.argv.slice(2)) { ...(gate.config ? ["--config", gate.config] : []), ...(gate.testPattern ? ["--testNamePattern", gate.testPattern] : []), "--reporter=default", "--reporter=json", `--outputFile.json=${file}`], - { cwd: resolve(root, gate.cwd), env, stdio: "inherit", timeout: 10 * 60_000 }); + { cwd: resolve(root, gate.cwd), + env: gate.id === "SH-1" ? { ...env, PAPERCLIP_STOCK_PREFLIGHT_RUNNERD: runnerdBinary } : env, + stdio: "inherit", timeout: 10 * 60_000 }); let report; try { report = JSON.parse(readFileSync(file, "utf8")); } catch { /* missing evidence fails closed below */ } results.push(gradeGate(gate, report, run.status)); diff --git a/tests/runner-e2e/stock-harness-checks.test.mjs b/tests/runner-e2e/stock-harness-checks.test.mjs index 4e073b2d43..428cfa9ac0 100644 --- a/tests/runner-e2e/stock-harness-checks.test.mjs +++ b/tests/runner-e2e/stock-harness-checks.test.mjs @@ -40,9 +40,9 @@ describe("stock harness prerequisite coverage", () => { }); describe("stock harness prerequisite admission", () => { - const current = { sha: "a".repeat(40), fingerprint: "b".repeat(64) }; + const current = { sha: "a".repeat(40), fingerprint: "b".repeat(64), runnerdSha256: "c".repeat(64) }; const receipt = () => ({ schema: "paperclip.stock-harness-preflight.v3", passed: true, - setup: { passed: true, exitCode: 0, sdkExitCode: 0, runnerdExitCode: 0 }, + setup: { passed: true, exitCode: 0, sdkExitCode: 0, runnerdExitCode: 0, runnerdSha256: current.runnerdSha256 }, providerCalls: 0, sourceSha: current.sha, sourceFingerprint: current.fingerprint, sourceErrors: [], gates: [...stockHarnessGates.map(g => g.id), "SH-1-rust"].map(id => ({ id, passed: true, exitCode: 0 })) }); it("admits the same passing source revision", () => expect(assertPreflightReceipt(receipt(), current).passed).toBe(true)); @@ -57,6 +57,7 @@ describe("stock harness prerequisite admission", () => { ["missing source", r => { r.sourceErrors.push("missing.ts"); }], ["failed cold setup", r => { r.setup.passed = false; r.setup.exitCode = 1; }], ["missing daemon build", r => { delete r.setup.runnerdExitCode; }], + ["changed daemon binary", r => { r.setup.runnerdSha256 = "d".repeat(64); }], ])("rejects %s before providers", (_name, mutate) => { const r = receipt(); mutate(r); expect(() => assertPreflightReceipt(r, current)).toThrow("exact source SHA and fingerprint"); });