fix(evals): link existing campaign summary when present

This commit is contained in:
Dotta committed 2026-10-27 12:24:44 -05:00
1 parent 3605d08784
commit 93d635a806
3 files changed
+42 -7

No files matched your search

+7 -6
View File
@@ -161,6 +161,12 @@ export async function regenerateRunnerDashboard(input: {
};
}),
);
const summary = await readFile(path.join(bundle, "summary.md"), "utf8").then(text => text.replace(
/^(Feedback across retained attempts:.*?); [0-9]+ distinct items awaiting review;/m, "$1;",
)).catch((error: NodeJS.ErrnoException) => {
if (error.code === "ENOENT") return undefined;
throw error;
});
const dashboard = renderRunnerE2EDashboard({
title: "Runner Full-Stack E2E",
generatedAt: normalized.generatedAt,
@@ -168,6 +174,7 @@ export async function regenerateRunnerDashboard(input: {
catalog: runnerMatrix,
entries,
campaign,
summaryAvailable: summary !== undefined,
history,
});
const upgraded = {
@@ -180,12 +187,6 @@ export async function regenerateRunnerDashboard(input: {
evidenceErrors: normalized.results[index]?.evidenceErrors ?? [],
})),
};
const summary = await readFile(path.join(bundle, "summary.md"), "utf8").then(text => text.replace(
/^(Feedback across retained attempts:.*?); [0-9]+ distinct items awaiting review;/m, "$1;",
)).catch((error: NodeJS.ErrnoException) => {
if (error.code === "ENOENT") return undefined;
throw error;
});
const reviewStatus = /^(?:[0-9]+ distinct feedback items await review\.|Feedback reviewed by (?:human|agent) at ).*$/m;
const refreshedSummary = summary === undefined ? undefined : reviewStatus.test(summary)
? summary.replace(reviewStatus, () => campaignFeedbackReviewText(campaign))
@@ -0,0 +1,32 @@
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { expect, it } from "vitest";
import { renderRunnerE2EDashboard } from "./dashboard.js";
import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
import { buildRunnerCampaign } from "./history.js";
const campaign = () => buildRunnerCampaign({ campaignId: "summary-link-fixture", generatedAt: "2026-10-27T00:00:00Z", expected: [], results: [] });
it("offers only the fixed relative summary link when a campaign summary is available", () => {
const input = { title: "Campaign", generatedAt: "2026-10-27T00:00:00Z", expected: [], catalog: [], entries: [], campaign: campaign() };
expect(renderRunnerE2EDashboard({ ...input, summaryAvailable: true })).toContain('<a href="summary.md">Campaign summary ↗</a>');
expect(renderRunnerE2EDashboard(input)).not.toContain('href="summary.md"');
expect(renderRunnerE2EDashboard({ ...input, campaign: undefined, summaryAvailable: true })).not.toContain('href="summary.md"');
});
it("regeneration derives link availability from the actual summary file without changing result identities", async () => {
const root = await mkdtemp(path.join(os.tmpdir(), "runner-summary-link-"));
try {
const before = campaign(); before.source.workflowRunUrl = "https://github.com/paperclipai/paperclip/actions/runs/123";
await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(before));
await writeFile(path.join(root, "summary.md"), "# Historical partial fixture\n\nSix incomplete or blocked journeys.\n");
await regenerateRunnerDashboard({ bundle: root, historyFile: null });
const html = await readFile(path.join(root, "index.html"), "utf8");
expect(html).toContain('href="summary.md"'); expect(html).toContain(before.source.workflowRunUrl);
const after = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8"));
expect(after.results).toEqual(before.results); expect(after.source).toEqual(before.source);
expect(after.selected).toBe(before.selected);
await rm(path.join(root, "summary.md"));
await regenerateRunnerDashboard({ bundle: root, historyFile: null });
expect(await readFile(path.join(root, "index.html"), "utf8")).not.toContain('href="summary.md"');
} finally { await rm(root, { recursive: true, force: true }); }
});
+3 -1
View File
@@ -44,6 +44,8 @@ export interface RunnerDashboardInput {
catalog: readonly ReportExecution[];
entries: readonly RunnerDashboardEntry[];
campaign?: RunnerE2ECampaign;
/** The bundle contains its existing relative summary.md document. */
summaryAvailable?: boolean;
history?: RunnerE2EHistoryIndex;
}
@@ -835,7 +837,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
<p class="eyebrow">Full-stack acceptance campaign</p>
<h1 tabindex="-1">${html(input.title)}</h1>
<p class="lede">Results across runner profiles and execution environments. Declared PNG screenshots and normalized results are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.</p>
${input.campaign ? `<p class="report-source"><span>${html(input.generatedAt)}</span>${input.campaign.source.sha ? `<code>${html(input.campaign.source.sha.slice(0,8))}</code>` : ""}${input.campaign.source.workflowRunUrl ? `<a href="${html(input.campaign.source.workflowRunUrl)}">Source run ↗</a>` : ""}</p>` : ""}
${input.campaign ? `<p class="report-source"><span>${html(input.generatedAt)}</span>${input.campaign.source.sha ? `<code>${html(input.campaign.source.sha.slice(0,8))}</code>` : ""}${input.campaign.source.workflowRunUrl ? `<a href="${html(input.campaign.source.workflowRunUrl)}">Source run ↗</a>` : ""}${input.summaryAvailable ? `<a href="summary.md">Campaign summary ↗</a>` : ""}</p>` : ""}
</div>
<div class="report-actions">
<div class="summary" aria-label="Campaign summary">