From b330ec27e219a608bb014c09708d4f4dce5cb032 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 15 Sep 2026 23:03:49 -0500 Subject: [PATCH] Preserve behavior failures when regenerating E2E dashboards Keep clean evidence distinct from a passing execution. Add a regression covering a failed result with evidenceValid=true. Co-Authored-By: Paperclip --- tests/runner-e2e/dashboard-regenerate.ts | 4 ++-- tests/runner-e2e/history.test.ts | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/dashboard-regenerate.ts b/tests/runner-e2e/dashboard-regenerate.ts index 0643656a9ae..e145c37bf89 100644 --- a/tests/runner-e2e/dashboard-regenerate.ts +++ b/tests/runner-e2e/dashboard-regenerate.ts @@ -131,8 +131,8 @@ export async function regenerateRunnerDashboard(input: { return { result, valid: - publishedResult.evidenceValid ?? - (result.status === "passed" && result.cleanup === "passed"), + result.status === "passed" && result.cleanup === "passed" && + publishedResult.evidenceValid !== false, errors: publishedResult.evidenceErrors ?? [], evidenceBaseHref, evidenceFiles, diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts index 7fcfd64ccb0..e44cb00ff36 100644 --- a/tests/runner-e2e/history.test.ts +++ b/tests/runner-e2e/history.test.ts @@ -101,6 +101,19 @@ describe("runner E2E campaign history", () => { expect(dashboard).not.toContain("NaN"); }); + it("does not turn a clean-evidence behavior failure into a pass when regenerating", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-failed-evidence-")); + temporaryDirectories.push(root); + const execution = runnerMatrix[0]; + const failedResult = { ...result(execution, "failed"), failureClass: "candidate_failure" as const, error: "Behavior check failed", evidenceValid: true, evidenceErrors: [] }; + const campaign = buildRunnerCampaign({ campaignId: "failed", generatedAt: failedResult.finishedAt, expected: [execution.id], results: [failedResult] }); + await writeFile(path.join(root, "normalized-results.json"), JSON.stringify({ ...campaign, results: [failedResult] })); + await regenerateRunnerDashboard({ bundle: root }); + const page = await readFile(path.join(root, "index.html"), "utf8"); + expect(page).toContain('class="case case-failed"'); + expect(page).not.toContain('class="case case-passed"'); + }); + it("records the resolved paid target instead of the trusted workflow checkout", () => { vi.stubEnv("PAPERCLIP_RUNNER_E2E_SOURCE_SHA", "target-sha"); vi.stubEnv("PAPERCLIP_RUNNER_E2E_SOURCE_REF", "refs/heads/target");