mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-05 12:29:37 +01:00
fix(evals): link existing campaign summary when present
This commit is contained in:
1 parent
3605d08784
commit
93d635a806
3 files changed
+42
-7
No files matched your search
@@ -161,6 +161,12 @@ export async function regenerateRunnerDashboard(input: {
|
||||
};
|
||||
}),
|
||||
);
|
||||
const summary = await readFile(path.join(bundle, "summary.md"), "utf8").then(text => text.replace(
|
||||
/^(Feedback across retained attempts:.*?); [0-9]+ distinct items awaiting review;/m, "$1;",
|
||||
)).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return undefined;
|
||||
throw error;
|
||||
});
|
||||
const dashboard = renderRunnerE2EDashboard({
|
||||
title: "Runner Full-Stack E2E",
|
||||
generatedAt: normalized.generatedAt,
|
||||
@@ -168,6 +174,7 @@ export async function regenerateRunnerDashboard(input: {
|
||||
catalog: runnerMatrix,
|
||||
entries,
|
||||
campaign,
|
||||
summaryAvailable: summary !== undefined,
|
||||
history,
|
||||
});
|
||||
const upgraded = {
|
||||
@@ -180,12 +187,6 @@ export async function regenerateRunnerDashboard(input: {
|
||||
evidenceErrors: normalized.results[index]?.evidenceErrors ?? [],
|
||||
})),
|
||||
};
|
||||
const summary = await readFile(path.join(bundle, "summary.md"), "utf8").then(text => text.replace(
|
||||
/^(Feedback across retained attempts:.*?); [0-9]+ distinct items awaiting review;/m, "$1;",
|
||||
)).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return undefined;
|
||||
throw error;
|
||||
});
|
||||
const reviewStatus = /^(?:[0-9]+ distinct feedback items await review\.|Feedback reviewed by (?:human|agent) at ).*$/m;
|
||||
const refreshedSummary = summary === undefined ? undefined : reviewStatus.test(summary)
|
||||
? summary.replace(reviewStatus, () => campaignFeedbackReviewText(campaign))
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { expect, it } from "vitest";
|
||||
import { renderRunnerE2EDashboard } from "./dashboard.js";
|
||||
import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
|
||||
import { buildRunnerCampaign } from "./history.js";
|
||||
|
||||
const campaign = () => buildRunnerCampaign({ campaignId: "summary-link-fixture", generatedAt: "2026-10-27T00:00:00Z", expected: [], results: [] });
|
||||
it("offers only the fixed relative summary link when a campaign summary is available", () => {
|
||||
const input = { title: "Campaign", generatedAt: "2026-10-27T00:00:00Z", expected: [], catalog: [], entries: [], campaign: campaign() };
|
||||
expect(renderRunnerE2EDashboard({ ...input, summaryAvailable: true })).toContain('<a href="summary.md">Campaign summary ↗</a>');
|
||||
expect(renderRunnerE2EDashboard(input)).not.toContain('href="summary.md"');
|
||||
expect(renderRunnerE2EDashboard({ ...input, campaign: undefined, summaryAvailable: true })).not.toContain('href="summary.md"');
|
||||
});
|
||||
it("regeneration derives link availability from the actual summary file without changing result identities", async () => {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "runner-summary-link-"));
|
||||
try {
|
||||
const before = campaign(); before.source.workflowRunUrl = "https://github.com/paperclipai/paperclip/actions/runs/123";
|
||||
await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(before));
|
||||
await writeFile(path.join(root, "summary.md"), "# Historical partial fixture\n\nSix incomplete or blocked journeys.\n");
|
||||
await regenerateRunnerDashboard({ bundle: root, historyFile: null });
|
||||
const html = await readFile(path.join(root, "index.html"), "utf8");
|
||||
expect(html).toContain('href="summary.md"'); expect(html).toContain(before.source.workflowRunUrl);
|
||||
const after = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8"));
|
||||
expect(after.results).toEqual(before.results); expect(after.source).toEqual(before.source);
|
||||
expect(after.selected).toBe(before.selected);
|
||||
await rm(path.join(root, "summary.md"));
|
||||
await regenerateRunnerDashboard({ bundle: root, historyFile: null });
|
||||
expect(await readFile(path.join(root, "index.html"), "utf8")).not.toContain('href="summary.md"');
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -44,6 +44,8 @@ export interface RunnerDashboardInput {
|
||||
catalog: readonly ReportExecution[];
|
||||
entries: readonly RunnerDashboardEntry[];
|
||||
campaign?: RunnerE2ECampaign;
|
||||
/** The bundle contains its existing relative summary.md document. */
|
||||
summaryAvailable?: boolean;
|
||||
history?: RunnerE2EHistoryIndex;
|
||||
}
|
||||
|
||||
@@ -835,7 +837,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
|
||||
<p class="eyebrow">Full-stack acceptance campaign</p>
|
||||
<h1 tabindex="-1">${html(input.title)}</h1>
|
||||
<p class="lede">Results across runner profiles and execution environments. Declared PNG screenshots and normalized results are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.</p>
|
||||
${input.campaign ? `<p class="report-source"><span>${html(input.generatedAt)}</span>${input.campaign.source.sha ? `<code>${html(input.campaign.source.sha.slice(0,8))}</code>` : ""}${input.campaign.source.workflowRunUrl ? `<a href="${html(input.campaign.source.workflowRunUrl)}">Source run ↗</a>` : ""}</p>` : ""}
|
||||
${input.campaign ? `<p class="report-source"><span>${html(input.generatedAt)}</span>${input.campaign.source.sha ? `<code>${html(input.campaign.source.sha.slice(0,8))}</code>` : ""}${input.campaign.source.workflowRunUrl ? `<a href="${html(input.campaign.source.workflowRunUrl)}">Source run ↗</a>` : ""}${input.summaryAvailable ? `<a href="summary.md">Campaign summary ↗</a>` : ""}</p>` : ""}
|
||||
</div>
|
||||
<div class="report-actions">
|
||||
<div class="summary" aria-label="Campaign summary">
|
||||
|
||||
Reference in new issue
Block a user