From 2dd8931eee8a84afa7786e0533a300d9b5149ba3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 2 Oct 2026 12:23:54 -0500 Subject: [PATCH] fix(evals): build the cold daemon before protocol prerequisites Co-Authored-By: Paperclip --- tests/runner-e2e/STOCK-HARNESS.md | 7 +++++++ tests/runner-e2e/stock-harness-checks.mjs | 19 +++++++++++++++---- .../runner-e2e/stock-harness-checks.test.mjs | 5 +++-- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/tests/runner-e2e/STOCK-HARNESS.md b/tests/runner-e2e/STOCK-HARNESS.md index f7b2e30d86..ba2b8fe2f5 100644 --- a/tests/runner-e2e/STOCK-HARNESS.md +++ b/tests/runner-e2e/STOCK-HARNESS.md @@ -142,6 +142,13 @@ the ordinary server dependency builder first, retains its output and exit status and refuses admission when setup fails. A cold legacy pilot must pass before the full matrix retry. +The [fixed legacy pilot](https://github.com/paperclipai/paperclip/actions/runs/37039240025) +at `4163dbfd0fd4d145bfa52b4d7f80eb59a362ee36` passed SDK setup, hire/shared +prompt/oracle checks and the Rust additive test. It stopped before providers: +the real daemon-frame test lacked the cold `paperclip-runnerd` binary. Setup now +builds that daemon from the locked Rust source before TypeScript gates, retains +`runnerd-build.txt`, and requires both setup exits in the admission receipt. + Dispatch the trusted workflow from `master`, with `target_branch` naming the same-repository candidate and an exact cell selector first. The workflow resolves that target once to an immutable SHA. Never dispatch target-controlled workflow diff --git a/tests/runner-e2e/stock-harness-checks.mjs b/tests/runner-e2e/stock-harness-checks.mjs index 3e3368d99f..e8e88a1d00 100644 --- a/tests/runner-e2e/stock-harness-checks.mjs +++ b/tests/runner-e2e/stock-harness-checks.mjs @@ -87,8 +87,9 @@ export function sourceFingerprint() { export function assertPreflightReceipt(report, current) { const expected = [...stockHarnessGates.map(gate => gate.id), "SH-1-rust"]; - if (report?.schema !== "paperclip.stock-harness-preflight.v2" || report.passed !== true || + if (report?.schema !== "paperclip.stock-harness-preflight.v3" || report.passed !== true || report.setup?.passed !== true || report.setup?.exitCode !== 0 || + report.setup?.sdkExitCode !== 0 || report.setup?.runnerdExitCode !== 0 || report.providerCalls !== 0 || report.sourceSha !== current.sha || report.sourceFingerprint !== current.fingerprint || report.sourceErrors?.length !== 0 || !Array.isArray(report.gates) || report.gates.length !== expected.length || @@ -136,11 +137,21 @@ export function main(args = process.argv.slice(2)) { cwd: root, env, encoding: "utf8", timeout: 5 * 60_000, }); writeFileSync(join(output, "setup.txt"), `${setupRun.stdout ?? ""}\n${setupRun.stderr ?? ""}`); - const setup = { passed: setupRun.status === 0, exitCode: setupRun.status }; + // The exact daemon-frame test uses the real local Rust daemon. Legacy cold + // cells do not download native artifacts, so compile it before TS discovery. + const runnerd = setupRun.status === 0 ? spawnSync("cargo", ["build", "--locked", + ...(args.includes("--allow-rust-network") ? [] : ["--offline"]), + "-p", "paperclip-runner-core", "--bin", "paperclip-runnerd"], { + cwd: join(root, "packages/paperclip-runner/runner"), env, encoding: "utf8", timeout: 10 * 60_000, + }) : null; + writeFileSync(join(output, "runnerd-build.txt"), `${runnerd?.stdout ?? ""}\n${runnerd?.stderr ?? ""}`); + const setup = { passed: setupRun.status === 0 && runnerd?.status === 0, + exitCode: setupRun.status !== 0 ? setupRun.status : runnerd?.status ?? null, + sdkExitCode: setupRun.status, runnerdExitCode: runnerd?.status ?? null }; if (!setup.passed) { const source = sourceFingerprint(); writeFileSync(join(output, "preflight.json"), JSON.stringify({ - schema: "paperclip.stock-harness-preflight.v2", sourceSha: git.stdout?.trim() || null, + schema: "paperclip.stock-harness-preflight.v3", sourceSha: git.stdout?.trim() || null, sourceFingerprint: source.fingerprint, measuredAt: new Date().toISOString(), providerCalls: 0, live: "not_run", passed: false, sourceErrors: source.sourceErrors, setup, gates: [], }, null, 2) + "\n"); @@ -167,7 +178,7 @@ export function main(args = process.argv.slice(2)) { writeFileSync(join(output, "rust.txt"), rustOutput); results.push({ id: "SH-1-rust", passed: rust.status === 0 && /test runtime_instructions_are_additive_for_codex_on_start_and_resume \.\.\. ok/.test(rustOutput), exitCode: rust.status }); const { fingerprint, sourceErrors } = sourceFingerprint(); - const report = { schema: "paperclip.stock-harness-preflight.v2", sourceSha: git.stdout?.trim() || null, + const report = { schema: "paperclip.stock-harness-preflight.v3", sourceSha: git.stdout?.trim() || null, sourceFingerprint: fingerprint, measuredAt: new Date().toISOString(), providerCalls: 0, live: "not_run", passed: git.status === 0 && sourceErrors.length === 0 && results.every(row => row.passed), sourceErrors, setup, gates: results }; writeFileSync(join(output, "preflight.json"), JSON.stringify(report, null, 2) + "\n"); diff --git a/tests/runner-e2e/stock-harness-checks.test.mjs b/tests/runner-e2e/stock-harness-checks.test.mjs index b3884139af..4e073b2d43 100644 --- a/tests/runner-e2e/stock-harness-checks.test.mjs +++ b/tests/runner-e2e/stock-harness-checks.test.mjs @@ -41,8 +41,8 @@ describe("stock harness prerequisite coverage", () => { describe("stock harness prerequisite admission", () => { const current = { sha: "a".repeat(40), fingerprint: "b".repeat(64) }; - const receipt = () => ({ schema: "paperclip.stock-harness-preflight.v2", passed: true, - setup: { passed: true, exitCode: 0 }, + const receipt = () => ({ schema: "paperclip.stock-harness-preflight.v3", passed: true, + setup: { passed: true, exitCode: 0, sdkExitCode: 0, runnerdExitCode: 0 }, providerCalls: 0, sourceSha: current.sha, sourceFingerprint: current.fingerprint, sourceErrors: [], gates: [...stockHarnessGates.map(g => g.id), "SH-1-rust"].map(id => ({ id, passed: true, exitCode: 0 })) }); it("admits the same passing source revision", () => expect(assertPreflightReceipt(receipt(), current).passed).toBe(true)); @@ -56,6 +56,7 @@ describe("stock harness prerequisite admission", () => { ["provider calls", r => { r.providerCalls = 1; }], ["missing source", r => { r.sourceErrors.push("missing.ts"); }], ["failed cold setup", r => { r.setup.passed = false; r.setup.exitCode = 1; }], + ["missing daemon build", r => { delete r.setup.runnerdExitCode; }], ])("rejects %s before providers", (_name, mutate) => { const r = receipt(); mutate(r); expect(() => assertPreflightReceipt(r, current)).toThrow("exact source SHA and fingerprint"); });