From 7e8b3e9b75dd5d9cefcb88ea465d55e22760b786 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 2 Oct 2026 12:59:07 -0500 Subject: [PATCH] fix(evals): retain prerequisites inside the campaign artifact root Co-Authored-By: Paperclip --- tests/runner-e2e/STOCK-HARNESS.md | 16 +++++++++++++ tests/runner-e2e/launch.ts | 23 ++++++++----------- .../runner-e2e/select-rerun-artifacts.test.ts | 13 +++++++++++ .../stock-harness-admission.test.ts | 9 ++++---- tests/runner-e2e/stock-harness-admission.ts | 6 +++-- tests/runner-e2e/stock-harness-checks.mjs | 6 +++-- 6 files changed, 51 insertions(+), 22 deletions(-) diff --git a/tests/runner-e2e/STOCK-HARNESS.md b/tests/runner-e2e/STOCK-HARNESS.md index c5df1e385c..e0e8e04af4 100644 --- a/tests/runner-e2e/STOCK-HARNESS.md +++ b/tests/runner-e2e/STOCK-HARNESS.md @@ -157,6 +157,22 @@ protocol fixture absent. It also stopped before paid providers. Setup uses the package's ordinary locked workspace `--bins` build, covering both the daemon and its fixture; verification binds both binaries to the retained receipt. +That final cold pilot passed all 537 prerequisites on GitHub and reached Claude +Sonnet 4.6. It saved a workspace `task-output.md` and completed the issue, while +the independent durable-document oracle found no Paperclip issue document. +The pinned skill's phrase "task document" does not explicitly name storage in +Paperclip; matched historical results must precede any regression attribution. +The task, instruction delivery, budget, and cleanup receipts remain retained. + +Its trusted merged report rejected the prerequisite folder alongside the campaign +root and synthesized a missing-result infrastructure error. Raw packaged cell +results were uploaded and remain inspectable. Prerequisites now live under the +exact campaign root; the unchanged trusted selector contract is calibrated in +the mandatory gate. The initial full matched campaigns at `f02d8d0df` and +`12c5433c6` retain their original raw results and publication outcomes. Any +reconstructed comparison must declare directory-layout recovery and preserve +every result, hash, original failure, and assertion. + Dispatch the trusted workflow from `master`, with `target_branch` naming the same-repository candidate and an exact cell selector first. The workflow resolves that target once to an immutable SHA. Never dispatch target-controlled workflow diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index b315ddd53d..2e28e2a975 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1086,17 +1086,17 @@ async function main() { // profiles remain discoverable, but cannot reach a provider. assertRunnerE2EPrerequisites(executions); assertNativeCompletionSelection(executions); - let nativeCampaignId: string | undefined; + const campaignId = cleanId( + process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? + `local-${new Date().toISOString().replace(/[:.]/g, "-")}`, + ); + const summaryDir = path.join(resultsRoot, campaignId); + await mkdir(summaryDir, { recursive: true }); if (executions.some(execution => execution.suite.id === "native-completion")) { - nativeCampaignId = cleanId(process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? - `local-${new Date().toISOString().replace(/[:.]/g, "-")}`); - const nativeCampaignDirectory = path.join(resultsRoot, nativeCampaignId); - await mkdir(nativeCampaignDirectory, { recursive: true }); - process.env[NATIVE_COMPLETION_PREFLIGHT_ENV] = prepareNativeCompletionPreflight(nativeCampaignDirectory); + process.env[NATIVE_COMPLETION_PREFLIGHT_ENV] = prepareNativeCompletionPreflight(summaryDir); } if (executions.some(execution => execution.suite.id === "stock-harness")) { - process.env[STOCK_PREFLIGHT_ENV] = prepareStockHarnessPreflight(); - + process.env[STOCK_PREFLIGHT_ENV] = prepareStockHarnessPreflight(summaryDir); } await loadLocalEnvironment(process.env); @@ -1119,12 +1119,7 @@ async function main() { ); } - const campaignId = nativeCampaignId ?? cleanId( - process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? - `local-${new Date().toISOString().replace(/[:.]/g, "-")}`, - ); - const summaryDir = path.join(resultsRoot, campaignId); - await mkdir(summaryDir, { recursive: true }); + await writeFile( path.join(summaryDir, "invocation-policy.json"), `${JSON.stringify( diff --git a/tests/runner-e2e/select-rerun-artifacts.test.ts b/tests/runner-e2e/select-rerun-artifacts.test.ts index b6793b2885..657fc2cea1 100644 --- a/tests/runner-e2e/select-rerun-artifacts.test.ts +++ b/tests/runner-e2e/select-rerun-artifacts.test.ts @@ -160,6 +160,19 @@ function singletonSelectionInput(paths: Awaited>) { } describe("runner E2E workflow rerun artifact selection", () => { + it("retains credential-free prerequisites inside the exact campaign root", async () => { + const paths = await fixture(); + const { artifactName, campaignName } = await addArtifact({ root: paths.artifactRoot, + executionId: RERUN, workflowAttempt: 2, status: "passed" }); + const prerequisite = path.join(paths.artifactRoot, artifactName, campaignName, + "stock-harness-prerequisites", "fixture", "preflight.json"); + await mkdir(path.dirname(prerequisite), { recursive: true }); + await writeFile(prerequisite, JSON.stringify({ passed: true, providerCalls: 0 })); + await selectRerunArtifacts(singletonSelectionInput(paths)); + expect(await readFile(path.join(paths.selectedRoot, artifactName, campaignName, + "stock-harness-prerequisites", "fixture", "preflight.json"), "utf8")) + .toContain('"providerCalls":0'); + }); it.each(["before", "after"])("ignores a queued placeholder %s a started replacement without hiding its failure", async order => { const paths = await fixture(); diff --git a/tests/runner-e2e/stock-harness-admission.test.ts b/tests/runner-e2e/stock-harness-admission.test.ts index 0872ae55e5..20a07639ae 100644 --- a/tests/runner-e2e/stock-harness-admission.test.ts +++ b/tests/runner-e2e/stock-harness-admission.test.ts @@ -5,6 +5,7 @@ import { CREDENTIAL_NAMES } from "./types.js"; vi.mock("node:child_process", () => ({ spawnSync: vi.fn() })); const spawn = vi.mocked(spawnSync); +const campaignDirectory = "/fixture/results/gha-1-1-stock-harness.fixture"; beforeEach(() => { vi.resetAllMocks(); vi.unstubAllEnvs(); }); describe("stock harness credential-free admission", () => { @@ -19,21 +20,21 @@ describe("stock harness credential-free admission", () => { }); it("fails before provider execution when the prerequisite subprocess fails", () => { spawn.mockReturnValue({ status: 1 } as ReturnType); - expect(() => prepareStockHarnessPreflight()).toThrow("before provider execution"); + expect(() => prepareStockHarnessPreflight(campaignDirectory)).toThrow("before provider execution"); expect(spawn).toHaveBeenCalledTimes(1); }); it("verifies retained evidence after a passing prerequisite subprocess", () => { spawn.mockReturnValueOnce({ status: 0 } as ReturnType); spawn.mockReturnValueOnce({ status: 0, stdout: '{"passed":true}' } as ReturnType); - const receipt = prepareStockHarnessPreflight(); - expect(receipt).toMatch(/stock-harness-preflight-.*\/preflight.json$/); + const receipt = prepareStockHarnessPreflight(campaignDirectory); + expect(receipt).toMatch(/^\/fixture\/results\/gha-1-1-stock-harness\.fixture\/stock-harness-prerequisites\/[^/]+\/preflight.json$/); expect(spawn.mock.calls[1]?.[1]).toContain(`--verify=${receipt}`); }); it("allows Cargo dependency resolution only on the disposable GitHub runner", () => { vi.stubEnv("GITHUB_ACTIONS", "true"); spawn.mockReturnValueOnce({ status: 0 } as ReturnType); spawn.mockReturnValueOnce({ status: 0, stdout: '{}' } as ReturnType); - prepareStockHarnessPreflight(); + prepareStockHarnessPreflight(campaignDirectory); expect(spawn.mock.calls[0]?.[1]).toContain("--allow-rust-network"); }); it("refuses failed receipt verification", () => { diff --git a/tests/runner-e2e/stock-harness-admission.ts b/tests/runner-e2e/stock-harness-admission.ts index 09ff72e8fc..c1a0e40761 100644 --- a/tests/runner-e2e/stock-harness-admission.ts +++ b/tests/runner-e2e/stock-harness-admission.ts @@ -23,8 +23,10 @@ export function verifyStockHarnessPreflight(receiptPath: string | undefined) { return JSON.parse(run.stdout) as Record; } -export function prepareStockHarnessPreflight() { - const output = path.join(root, "tests/runner-e2e/results", `stock-harness-preflight-${randomUUID()}`); +export function prepareStockHarnessPreflight(campaignDirectory: string) { + // The trusted artifact selector admits exactly one campaign root. Keep all + // prerequisite evidence inside that root, including pre-provider failures. + const output = path.join(campaignDirectory, "stock-harness-prerequisites", randomUUID()); const receipt = path.join(output, "preflight.json"); const run = spawnSync(process.execPath, [path.join(root, "tests/runner-e2e/stock-harness-checks.mjs"), `--output-dir=${output}`, ...(process.env.GITHUB_ACTIONS === "true" ? ["--allow-rust-network"] : [])], { diff --git a/tests/runner-e2e/stock-harness-checks.mjs b/tests/runner-e2e/stock-harness-checks.mjs index 7fcd16b364..ec89e0ee5e 100644 --- a/tests/runner-e2e/stock-harness-checks.mjs +++ b/tests/runner-e2e/stock-harness-checks.mjs @@ -41,11 +41,13 @@ export const stockHarnessGates = [ { id: "SH-eval", name: "Independent oracle and qualification admission", cwd: ".", config: "tests/runner-e2e/vitest.config.ts", files: ["tests/runner-e2e/stock-harness.test.ts", "tests/runner-e2e/stock-harness-checks.test.mjs", - "tests/runner-e2e/stock-harness-admission.test.ts", "tests/runner-e2e/stock-harness-digest.test.ts"], + "tests/runner-e2e/stock-harness-admission.test.ts", "tests/runner-e2e/stock-harness-digest.test.ts", + "tests/runner-e2e/select-rerun-artifacts.test.ts"], required: ["rejects old SHA before providers", "allows toolchain paths and excludes every present or future credential", "changes when the evaluated server/src/onboarding-assets/default/AGENTS.md changes", "changes when the evaluated packages/adapter-utils/src/server-utils.ts changes", - "changes when the evaluated packages/shared/src/connection-intent-guidance.ts changes"] }, + "changes when the evaluated packages/shared/src/connection-intent-guidance.ts changes", + "retains credential-free prerequisites inside the exact campaign root"] }, ]; export function gradeGate(gate, report, exitCode) {