mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-25 17:45:53 +01:00
316 lines
13 KiB
TypeScript
316 lines
13 KiB
TypeScript
import type { MatrixExecution, RunnerE2EResult } from "./types.js";
|
|
|
|
export interface RunnerDashboardEntry {
|
|
result: RunnerE2EResult;
|
|
valid: boolean;
|
|
errors: readonly string[];
|
|
evidenceBaseHref?: string;
|
|
evidenceFiles?: readonly string[];
|
|
}
|
|
|
|
export interface RunnerDashboardInput {
|
|
title: string;
|
|
generatedAt: string;
|
|
expected: readonly string[];
|
|
catalog: readonly MatrixExecution[];
|
|
entries: readonly RunnerDashboardEntry[];
|
|
}
|
|
|
|
function html(value: unknown) {
|
|
return String(value ?? "")
|
|
.replaceAll("&", "&")
|
|
.replaceAll("<", "<")
|
|
.replaceAll(">", ">")
|
|
.replaceAll('"', """)
|
|
.replaceAll("'", "'");
|
|
}
|
|
|
|
function durationLabel(durationMs: number) {
|
|
if (durationMs < 1_000) return `${durationMs}ms`;
|
|
const seconds = Math.round(durationMs / 1_000);
|
|
if (seconds < 60) return `${seconds}s`;
|
|
return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
|
|
}
|
|
|
|
function safeEvidenceHref(base: string | undefined, relative: string) {
|
|
if (!base || /^(?:[a-z]+:|\/\/)/i.test(base)) return null;
|
|
const cleanBase = base
|
|
.split("/")
|
|
.filter((segment) => segment && segment !== "." && segment !== "..")
|
|
.map(encodeURIComponent)
|
|
.join("/");
|
|
const cleanRelative = relative
|
|
.split("/")
|
|
.filter((segment) => segment && segment !== "." && segment !== "..")
|
|
.map(encodeURIComponent)
|
|
.join("/");
|
|
return `${cleanBase}/${cleanRelative}`;
|
|
}
|
|
|
|
function compactJson(value: unknown) {
|
|
const serialized = JSON.stringify(value);
|
|
return serialized && serialized.length > 1_200
|
|
? `${serialized.slice(0, 1_200)}…`
|
|
: serialized;
|
|
}
|
|
|
|
function renderCase(
|
|
execution: MatrixExecution,
|
|
expected: ReadonlySet<string>,
|
|
entryById: ReadonlyMap<string, RunnerDashboardEntry>,
|
|
) {
|
|
const selected = expected.has(execution.id);
|
|
const entry = entryById.get(execution.id);
|
|
const state = !selected
|
|
? "not-selected"
|
|
: !entry
|
|
? "missing"
|
|
: entry.valid
|
|
? "passed"
|
|
: "failed";
|
|
const label = state.replace("-", " ");
|
|
const detail =
|
|
entry?.errors.join("; ") ||
|
|
(entry?.valid ? "All invariants passed" : "Not selected");
|
|
const declaredScreenshots = entry?.result.screenshots?.length
|
|
? entry.result.screenshots
|
|
: entry
|
|
? [
|
|
{
|
|
id: "final-state",
|
|
label: "Final visible task state",
|
|
file: "final-state.png",
|
|
},
|
|
]
|
|
: [];
|
|
const availableFiles = entry?.evidenceFiles
|
|
? new Set(entry.evidenceFiles)
|
|
: null;
|
|
const screenshots = declaredScreenshots
|
|
.filter((item) => !availableFiles || availableFiles.has(item.file))
|
|
.flatMap((item) => {
|
|
const href = safeEvidenceHref(entry?.evidenceBaseHref, item.file);
|
|
return href ? [{ ...item, href }] : [];
|
|
});
|
|
if (
|
|
entry?.result.status === "failed" &&
|
|
(!availableFiles || availableFiles.has("failure.png"))
|
|
) {
|
|
const href = safeEvidenceHref(entry.evidenceBaseHref, "failure.png");
|
|
if (href)
|
|
screenshots.push({
|
|
id: "failure",
|
|
label: "Failure state",
|
|
file: "failure.png",
|
|
href,
|
|
});
|
|
}
|
|
const playwright = safeEvidenceHref(
|
|
entry?.evidenceBaseHref,
|
|
"html-report/index.html",
|
|
);
|
|
const links =
|
|
screenshots.length > 0 || playwright
|
|
? `<nav aria-label="Evidence for ${html(execution.id)}">
|
|
${screenshots.map((item) => `<a href="${html(item.href)}">${html(item.label)}</a>`).join("")}
|
|
${playwright ? `<a href="${html(playwright)}">Playwright report</a>` : ""}
|
|
</nav>`
|
|
: "";
|
|
const matcherRows = (entry?.result.matcherResults ?? [])
|
|
.map(
|
|
(result) => `<tr class="matcher-${result.passed ? "passed" : "failed"}">
|
|
<td>${result.passed ? "✓" : "✕"}</td>
|
|
<td><code>${html(result.matcher.kind)}</code></td>
|
|
<td>${html(compactJson(result.matcher))}</td>
|
|
<td>${html(result.detail)}</td>
|
|
</tr>`,
|
|
)
|
|
.join("");
|
|
const gallery = screenshots.length
|
|
? `<div class="gallery">${screenshots
|
|
.map(
|
|
(item) => `<a href="${html(item.href)}" title="${html(item.label)}">
|
|
<img src="${html(item.href)}" loading="lazy" alt="${html(item.label)} for ${html(execution.id)}">
|
|
<span>${html(item.label)}</span>
|
|
</a>`,
|
|
)
|
|
.join("")}</div>`
|
|
: "";
|
|
return `<article class="case case-${state}" data-execution-id="${html(execution.id)}">
|
|
<div class="case-heading">
|
|
<strong>${html(execution.task.label)}</strong>
|
|
<span class="status">${html(label)}</span>
|
|
</div>
|
|
${gallery}
|
|
<code>${html(execution.id)}</code>
|
|
<details class="case-context">
|
|
<summary>Matchers and test context</summary>
|
|
${
|
|
entry
|
|
? `<dl>
|
|
<div><dt>Attempt</dt><dd>${entry.result.attempt}</dd></div>
|
|
<div><dt>Duration</dt><dd>${html(durationLabel(entry.result.durationMs))}</dd></div>
|
|
<div><dt>Runtime</dt><dd>${html(entry.result.runtimeMode)}</dd></div>
|
|
<div><dt>Provider</dt><dd>${html(entry.result.provider)}</dd></div>
|
|
<div><dt>Model</dt><dd>${html(entry.result.model)}</dd></div>
|
|
${entry.result.issueIdentifier ? `<div><dt>Issue</dt><dd>${html(entry.result.issueIdentifier)}</dd></div>` : ""}
|
|
</dl>`
|
|
: ""
|
|
}
|
|
<p class="detail">${html(detail)}</p>
|
|
${matcherRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Pass</th><th>Matcher</th><th>Expectation</th><th>Detail</th></tr></thead><tbody>${matcherRows}</tbody></table></div>` : `<p class="detail">No matcher result was recorded.</p>`}
|
|
${entry?.result.usage ? `<details class="usage"><summary>Usage and cost metadata</summary><pre>${html(JSON.stringify(entry.result.usage, null, 2))}</pre></details>` : ""}
|
|
${links}
|
|
</details>
|
|
</article>`;
|
|
}
|
|
|
|
export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
|
|
const expected = new Set(input.expected);
|
|
const entryById = new Map(
|
|
input.entries.map((entry) => [entry.result.executionId, entry]),
|
|
);
|
|
const profiles = [
|
|
...new Map(
|
|
input.catalog.map((execution) => [
|
|
execution.profile.id,
|
|
execution.profile,
|
|
]),
|
|
).values(),
|
|
];
|
|
const environments = [
|
|
...new Map(
|
|
input.catalog.map((execution) => [
|
|
execution.environment.id,
|
|
execution.environment,
|
|
]),
|
|
).values(),
|
|
];
|
|
const selectedEntries = input.entries.filter((entry) =>
|
|
expected.has(entry.result.executionId),
|
|
);
|
|
const passed = selectedEntries.filter((entry) => entry.valid).length;
|
|
const failed = input.expected.length - passed;
|
|
const totalDuration = selectedEntries.reduce(
|
|
(total, entry) => total + entry.result.durationMs,
|
|
0,
|
|
);
|
|
const rows = profiles
|
|
.map((profile) => {
|
|
const columns = environments
|
|
.map((environment) => {
|
|
const executions = input.catalog.filter(
|
|
(execution) =>
|
|
execution.profile.id === profile.id &&
|
|
execution.environment.id === environment.id,
|
|
);
|
|
return `<td>${executions
|
|
.map((execution) => renderCase(execution, expected, entryById))
|
|
.join("")}</td>`;
|
|
})
|
|
.join("");
|
|
return `<tr>
|
|
<th scope="row">
|
|
<strong>${html(profile.label)}</strong>
|
|
<span class="generation">${html(profile.generation)}</span>
|
|
<small>${html(profile.provider)} · ${html(profile.model)}</small>
|
|
</th>
|
|
${columns}
|
|
</tr>`;
|
|
})
|
|
.join("");
|
|
const environmentHeaders = environments
|
|
.map(
|
|
(environment) =>
|
|
`<th scope="col"><strong>${html(environment.label)}</strong><small>${html(environment.provider)} · ${html(environment.expectedExecutionTarget.kind)}</small></th>`,
|
|
)
|
|
.join("");
|
|
|
|
return `<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<meta name="color-scheme" content="dark light">
|
|
<title>${html(input.title)}</title>
|
|
<style>
|
|
:root { color-scheme: dark; --bg: #0b1020; --panel: #131a2d; --line: #2a3554; --text: #eef2ff; --muted: #9aa8c7; --pass: #3ddc97; --fail: #ff6b81; --missing: #ffbd59; --idle: #69738c; }
|
|
* { box-sizing: border-box; }
|
|
body { margin: 0; background: radial-gradient(circle at top left, #18264a 0, var(--bg) 42rem); color: var(--text); font: 14px/1.45 ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
|
|
main { width: min(1500px, calc(100% - 32px)); margin: 32px auto 64px; }
|
|
header { display: flex; align-items: end; justify-content: space-between; gap: 24px; margin-bottom: 24px; }
|
|
h1 { margin: 0 0 4px; font-size: clamp(24px, 4vw, 42px); letter-spacing: -0.04em; }
|
|
header p, small, .detail { color: var(--muted); }
|
|
.summary { display: flex; gap: 10px; flex-wrap: wrap; justify-content: flex-end; }
|
|
.metric { min-width: 92px; padding: 10px 13px; border: 1px solid var(--line); border-radius: 12px; background: color-mix(in srgb, var(--panel) 88%, transparent); }
|
|
.metric strong { display: block; font-size: 20px; }
|
|
.metric span { color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: .08em; }
|
|
.table-wrap { overflow-x: auto; border: 1px solid var(--line); border-radius: 16px; background: color-mix(in srgb, var(--panel) 82%, transparent); box-shadow: 0 24px 80px #0006; }
|
|
table { width: 100%; min-width: 960px; border-collapse: collapse; }
|
|
th, td { padding: 14px; border-right: 1px solid var(--line); border-bottom: 1px solid var(--line); vertical-align: top; text-align: left; }
|
|
tr:last-child th, tr:last-child td { border-bottom: 0; }
|
|
th:last-child, td:last-child { border-right: 0; }
|
|
thead th { background: #11182a; }
|
|
thead small, tbody th small { display: block; margin-top: 3px; font-weight: 400; }
|
|
tbody th { width: 250px; background: #101729; }
|
|
.generation { display: inline-block; margin-left: 7px; padding: 2px 7px; border: 1px solid var(--line); border-radius: 999px; color: var(--muted); font-size: 10px; text-transform: uppercase; letter-spacing: .08em; }
|
|
.case { min-height: 170px; padding: 13px; border: 1px solid var(--line); border-left-width: 4px; border-radius: 12px; background: #0e1527cc; }
|
|
.case-passed { border-left-color: var(--pass); }
|
|
.case-failed { border-left-color: var(--fail); }
|
|
.case-missing { border-left-color: var(--missing); }
|
|
.case-not-selected { border-left-color: var(--idle); opacity: .62; }
|
|
.case-heading { display: flex; align-items: start; justify-content: space-between; gap: 10px; }
|
|
.status { padding: 3px 8px; border-radius: 999px; background: #27314b; font-size: 10px; font-weight: 800; text-transform: uppercase; letter-spacing: .07em; }
|
|
.case-passed .status { color: var(--pass); }
|
|
.case-failed .status { color: var(--fail); }
|
|
.case-missing .status { color: var(--missing); }
|
|
code { display: block; margin: 9px 0; color: #bec9e7; font-size: 11px; overflow-wrap: anywhere; }
|
|
dl { display: flex; flex-wrap: wrap; gap: 5px 14px; margin: 8px 0; }
|
|
dl div { display: flex; gap: 5px; }
|
|
dt { color: var(--muted); }
|
|
dd { margin: 0; }
|
|
.detail { margin: 10px 0; font-size: 12px; overflow-wrap: anywhere; }
|
|
nav { display: flex; flex-wrap: wrap; gap: 10px; }
|
|
a { color: #8bb8ff; font-size: 12px; }
|
|
.gallery { display: grid; grid-template-columns: repeat(auto-fit, minmax(130px, 1fr)); gap: 8px; margin: 12px 0; }
|
|
.gallery a { display: block; text-decoration: none; }
|
|
.gallery img { display: block; width: 100%; aspect-ratio: 16 / 10; object-fit: cover; object-position: top; border: 1px solid var(--line); border-radius: 8px; background: #080c16; }
|
|
.gallery span { display: block; margin-top: 4px; color: var(--muted); font-size: 10px; }
|
|
.matcher-wrap { overflow-x: auto; margin: 10px 0; }
|
|
.matchers { min-width: 650px; font-size: 11px; }
|
|
.matchers th, .matchers td { padding: 7px; border: 1px solid var(--line); }
|
|
.matchers .matcher-passed td:first-child { color: var(--pass); }
|
|
.matchers .matcher-failed td:first-child { color: var(--fail); }
|
|
.usage { margin: 10px 0; color: var(--muted); }
|
|
.case-context > summary { cursor: pointer; color: #8bb8ff; font-size: 12px; }
|
|
pre { max-height: 220px; overflow: auto; padding: 10px; border-radius: 8px; background: #080c16; color: #bec9e7; font-size: 10px; white-space: pre-wrap; }
|
|
footer { margin-top: 14px; color: var(--muted); font-size: 12px; }
|
|
@media (max-width: 760px) { header { align-items: start; flex-direction: column; } .summary { justify-content: start; } }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<main>
|
|
<header>
|
|
<div>
|
|
<h1>${html(input.title)}</h1>
|
|
<p>Paid full-stack runner acceptance matrix</p>
|
|
</div>
|
|
<div class="summary" aria-label="Campaign summary">
|
|
<div class="metric"><strong>${passed}/${input.expected.length}</strong><span>Passed</span></div>
|
|
<div class="metric"><strong>${failed}</strong><span>Failed</span></div>
|
|
<div class="metric"><strong>${html(durationLabel(totalDuration))}</strong><span>Test time</span></div>
|
|
</div>
|
|
</header>
|
|
<div class="table-wrap">
|
|
<table>
|
|
<thead><tr><th scope="col">Agent profile</th>${environmentHeaders}</tr></thead>
|
|
<tbody>${rows}</tbody>
|
|
</table>
|
|
</div>
|
|
<footer>Generated ${html(input.generatedAt)} · ${input.catalog.length} catalog executions · ${input.expected.length} selected</footer>
|
|
</main>
|
|
</body>
|
|
</html>
|
|
`;
|
|
}
|