import type { MatrixExecution, RunnerE2EResult } from "./types.js"; export interface RunnerDashboardEntry { result: RunnerE2EResult; valid: boolean; errors: readonly string[]; evidenceBaseHref?: string; evidenceFiles?: readonly string[]; } export interface RunnerDashboardInput { title: string; generatedAt: string; expected: readonly string[]; catalog: readonly MatrixExecution[]; entries: readonly RunnerDashboardEntry[]; } function html(value: unknown) { return String(value ?? "") .replaceAll("&", "&") .replaceAll("<", "<") .replaceAll(">", ">") .replaceAll('"', """) .replaceAll("'", "'"); } function durationLabel(durationMs: number) { if (durationMs < 1_000) return `${durationMs}ms`; const seconds = Math.round(durationMs / 1_000); if (seconds < 60) return `${seconds}s`; return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; } function safeEvidenceHref(base: string | undefined, relative: string) { if (!base || /^(?:[a-z]+:|\/\/)/i.test(base)) return null; const cleanBase = base .split("/") .filter((segment) => segment && segment !== "." && segment !== "..") .map(encodeURIComponent) .join("/"); const cleanRelative = relative .split("/") .filter((segment) => segment && segment !== "." && segment !== "..") .map(encodeURIComponent) .join("/"); return `${cleanBase}/${cleanRelative}`; } function compactJson(value: unknown) { const serialized = JSON.stringify(value); return serialized && serialized.length > 1_200 ? `${serialized.slice(0, 1_200)}…` : serialized; } function renderCase( execution: MatrixExecution, expected: ReadonlySet, entryById: ReadonlyMap, ) { const selected = expected.has(execution.id); const entry = entryById.get(execution.id); const state = !selected ? "not-selected" : !entry ? "missing" : entry.valid ? "passed" : "failed"; const label = state.replace("-", " "); const detail = entry?.errors.join("; ") || (entry?.valid ? "All invariants passed" : "Not selected"); const declaredScreenshots = entry?.result.screenshots?.length ? entry.result.screenshots : entry ? [ { id: "final-state", label: "Final visible task state", file: "final-state.png", }, ] : []; const availableFiles = entry?.evidenceFiles ? new Set(entry.evidenceFiles) : null; const screenshots = declaredScreenshots .filter((item) => !availableFiles || availableFiles.has(item.file)) .flatMap((item) => { const href = safeEvidenceHref(entry?.evidenceBaseHref, item.file); return href ? [{ ...item, href }] : []; }); if ( entry?.result.status === "failed" && (!availableFiles || availableFiles.has("failure.png")) ) { const href = safeEvidenceHref(entry.evidenceBaseHref, "failure.png"); if (href) screenshots.push({ id: "failure", label: "Failure state", file: "failure.png", href, }); } const playwright = safeEvidenceHref( entry?.evidenceBaseHref, "html-report/index.html", ); const links = screenshots.length > 0 || playwright ? `` : ""; const matcherRows = (entry?.result.matcherResults ?? []) .map( (result) => ` ${result.passed ? "✓" : "✕"} ${html(result.matcher.kind)} ${html(compactJson(result.matcher))} ${html(result.detail)} `, ) .join(""); const gallery = screenshots.length ? `

${screenshots .map( (item) => `

${html(item.label)} for ${html(execution.id)}

${html(item.label)} `, ) .join("")}

` : ""; return `

${html(execution.task.label)} ${html(label)}

${gallery} ${html(execution.id)}

Matchers and test context

${ entry ? `

Attempt: ${entry.result.attempt}
Duration: ${html(durationLabel(entry.result.durationMs))}
Runtime: ${html(entry.result.runtimeMode)}
Provider: ${html(entry.result.provider)}
Model: ${html(entry.result.model)}
Issue: ${html(entry.result.issueIdentifier)}

` : "" }

${html(detail)}

${matcherRows ? `

${matcherRows}

Pass	Matcher	Expectation	Detail

` : `

No matcher result was recorded.

`} ${entry?.result.usage ? `

Usage and cost metadata

${html(JSON.stringify(entry.result.usage, null, 2))}

` : ""} ${links}

`; } export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { const expected = new Set(input.expected); const entryById = new Map( input.entries.map((entry) => [entry.result.executionId, entry]), ); const profiles = [ ...new Map( input.catalog.map((execution) => [ execution.profile.id, execution.profile, ]), ).values(), ]; const environments = [ ...new Map( input.catalog.map((execution) => [ execution.environment.id, execution.environment, ]), ).values(), ]; const selectedEntries = input.entries.filter((entry) => expected.has(entry.result.executionId), ); const passed = selectedEntries.filter((entry) => entry.valid).length; const failed = input.expected.length - passed; const totalDuration = selectedEntries.reduce( (total, entry) => total + entry.result.durationMs, 0, ); const rows = profiles .map((profile) => { const columns = environments .map((environment) => { const executions = input.catalog.filter( (execution) => execution.profile.id === profile.id && execution.environment.id === environment.id, ); return `${executions .map((execution) => renderCase(execution, expected, entryById)) .join("")}`; }) .join(""); return ` ${html(profile.label)} ${html(profile.generation)} ${html(profile.provider)} · ${html(profile.model)} ${columns} `; }) .join(""); const environmentHeaders = environments .map( (environment) => `${html(environment.label)}${html(environment.provider)} · ${html(environment.expectedExecutionTarget.kind)}`, ) .join(""); return ` ${html(input.title)}

${html(input.title)}

Paid full-stack runner acceptance matrix

${passed}/${input.expected.length}Passed

${failed}Failed

${html(durationLabel(totalDuration))}Test time

${environmentHeaders}${rows}

Agent profile

`; }