diff --git a/tests/e2e/runner-e2e-dashboard.spec.ts b/tests/e2e/runner-e2e-dashboard.spec.ts index f29ca3b04b7..e9e1d8e550e 100644 --- a/tests/e2e/runner-e2e-dashboard.spec.ts +++ b/tests/e2e/runner-e2e-dashboard.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from "@playwright/test"; +import { expect, test, type Page } from "@playwright/test"; import { runnerExecutionById, runnerMatrix } from "../runner-e2e/catalog.js"; import { @@ -8,6 +8,13 @@ import { import { buildRunnerCampaign } from "../runner-e2e/history.js"; import type { MatrixExecution, RunnerE2EResult } from "../runner-e2e/types.js"; +const reportUrl = "http://runner-e2e.test/campaign/"; + +async function openReport(page: Page, report: string) { + await page.route(`${reportUrl}**`, (route) => route.fulfill({ contentType: "text/html", body: report })); + await page.goto(reportUrl); +} + const executionIds = [ "core-compatibility.legacy-codex.local.message-marker", "core-compatibility.legacy-codex.daytona.message-marker", @@ -66,14 +73,15 @@ function resultFor( }; } -test("filters matrix results and pages through the filtered screenshot gallery", async ({ +test("filters execution previews and pages through the filtered screenshot gallery", async ({ page, }) => { const catalog = executionIds.map(runnerExecutionById); const entries = catalog.map((execution, index) => resultFor(execution, index === 1 ? "failed" : "passed"), ); - await page.setContent( + await openReport( + page, renderRunnerE2EDashboard({ title: "Runner E2E interaction fixture", generatedAt: "2026-09-05T12:01:00.000Z", @@ -92,7 +100,7 @@ test("filters matrix results and pages through the filtered screenshot gallery", await expect(page.locator('select[data-report-suite] option')).toHaveCount(3); await expect(page.locator('select[data-report-profile] option')).toHaveCount(4); await expect(page.locator("[data-report-result]")).toHaveText( - "4 of 4 tests shown", + "4 of 4 executions shown", ); await expect(page.locator("[data-gallery-item]")).toHaveCount(4); const passedCard = page.locator('[data-execution-id="core-compatibility.legacy-codex.local.message-marker"]'); @@ -110,32 +118,23 @@ test("filters matrix results and pages through the filtered screenshot gallery", await page.locator('.feature-coverage').getByRole('button', { name: 'Questions and approvals', exact: true }).click(); await expect(page.locator('select[data-report-feature]')).toHaveValue('questions-and-approvals'); await expect(page.locator('.feature-coverage')).not.toHaveAttribute('open'); - await expect(page.locator('[data-report-result]')).toHaveText('1 of 4 tests shown'); + await expect(page.locator('[data-report-result]')).toHaveText('1 of 4 executions shown'); await expect(page.locator('.suite-section:visible')).toHaveCount(1); await expect(page.locator('[data-gallery-open]')).toHaveText('View gallery · 1'); await page.setViewportSize({ width: 900, height: 900 }); - await expect(page.locator('[data-report-profile-row]:visible')).toHaveCount(1); + await expect(page.locator('.execution-preview:visible')).toHaveCount(1); await page.setViewportSize({ width: 1280, height: 720 }); await page.locator('[data-report-reset]').click(); - const firstTable = page.locator(".matrix").first(); - const widthsBefore = await firstTable - .locator("thead th") - .evaluateAll((cells) => - cells.map((cell) => cell.getBoundingClientRect().width), - ); - await firstTable.locator("details summary").first().click(); - const widthsAfter = await firstTable - .locator("thead th") - .evaluateAll((cells) => - cells.map((cell) => cell.getBoundingClientRect().width), - ); - expect(widthsAfter).toEqual(widthsBefore); + const widthBefore = await passedCard.evaluate((card) => card.getBoundingClientRect().width); + await passedCard.locator(".case-diagnostics > summary").click(); + const widthAfter = await passedCard.evaluate((card) => card.getBoundingClientRect().width); + expect(widthAfter).toBe(widthBefore); await expect(passedCard.locator(".billing-strip")).toBeVisible(); await page.locator("[data-report-query]").fill("Legacy Claude"); await expect(page.locator("[data-report-result]")).toHaveText( - "1 of 4 tests shown", + "1 of 4 executions shown", ); await expect(page.locator("[data-gallery-open]")).toHaveText( "View gallery · 1", @@ -147,7 +146,7 @@ test("filters matrix results and pages through the filtered screenshot gallery", .locator("select[data-report-profile]") .selectOption("legacy-codex"); await expect(page.locator("[data-report-result]")).toHaveText( - "2 of 4 tests shown", + "2 of 4 executions shown", ); await expect(page.locator("[data-gallery-open]")).toHaveText( "View gallery · 2", @@ -206,13 +205,11 @@ function navigationReport(ids: readonly string[], suppliedEntries?: readonly Run test("suite links reveal filtered sections and scroll on repeated clicks", async ({ page }) => { const report = navigationReport(executionIds); - const reportUrl = "http://runner-e2e.test/campaign/"; - await page.route(reportUrl, (route) => route.fulfill({ contentType: "text/html", body: report })); - await page.goto(reportUrl); + await openReport(page, report); const core = page.locator("#suite-core-compatibility"); const coreHeading = core.getByRole("heading", { name: "Core Runner Compatibility", exact: true }); - const coreLink = page.getByRole("link", { name: "Core Runner Compatibility", exact: true }); + const coreLink = page.locator('[data-nav-suite-link="core-compatibility"]'); const backToTop = page.getByRole("link", { name: "Back to top", exact: true }); await expect(backToTop).toBeHidden(); const feature = page.locator("select[data-report-feature]"); @@ -223,8 +220,8 @@ test("suite links reveal filtered sections and scroll on repeated clicks", async await expect(core).toBeVisible(); await expect(coreHeading).toBeInViewport(); await expect(feature).toHaveValue(""); - await expect(page).toHaveURL(`${reportUrl}#suite-core-compatibility`); - await expect(page.locator("[data-report-result]")).toHaveText("4 of 4 tests shown"); + await expect(page).toHaveURL(`${reportUrl}?view=all#suite-core-compatibility`); + await expect(page.locator("[data-report-result]")).toHaveText("4 of 4 executions shown"); } await page.locator("select[data-report-profile]").selectOption("legacy-codex"); @@ -241,7 +238,7 @@ test("suite links reveal filtered sections and scroll on repeated clicks", async await expect(page.locator("select[data-report-profile]")).toHaveValue("legacy-codex"); await coreLink.click(); - await page.getByRole("link", { name: "Local Session Integrity", exact: true }).click(); + await page.locator('[data-nav-suite-link="local-session-integrity"]').click(); await expect(page.locator("#suite-local-session-integrity")).toBeVisible(); await feature.selectOption("questions-and-approvals"); await page.goBack(); @@ -249,50 +246,69 @@ test("suite links reveal filtered sections and scroll on repeated clicks", async await expect(feature).toHaveValue(""); }); -test("folds the suite index and links each case to a matching configuration", async ({ page }) => { +test("folds the suite tree, inspects configurations, and returns to the scrolling report", async ({ page }) => { const ids = [...executionIds, "core-compatibility.legacy-codex.local.ask-question"]; - const reportUrl = "http://runner-e2e.test/campaign/"; - await page.route(reportUrl, (route) => route.fulfill({ contentType: "text/html", body: navigationReport(ids) })); - await page.goto(reportUrl); - const coreIndex = page.locator(".campaign-suite-details").filter({ - has: page.getByRole("link", { name: "Core Runner Compatibility", exact: true }), - }); - await expect(coreIndex).not.toHaveAttribute("open"); - await coreIndex.getByRole("link", { name: "Core Runner Compatibility", exact: true }).click(); + await openReport(page, navigationReport(ids)); + const coreIndex = page.locator('[data-nav-suite="core-compatibility"]'); + const coreLink = coreIndex.locator('[data-nav-suite-link]'); await expect(coreIndex).not.toHaveAttribute("open"); + await coreLink.click(); + await expect(coreIndex).toHaveAttribute("open"); await coreIndex.locator("summary").focus(); await page.keyboard.press("Enter"); + await expect(coreIndex).not.toHaveAttribute("open"); + await page.keyboard.press("Enter"); await expect(coreIndex).toHaveAttribute("open"); - await expect(coreIndex.locator(".suite-case-list a")).toHaveCount(2); + await expect(coreIndex.locator("[data-nav-case]")).toHaveCount(2); const sectionToc = page.locator("#suite-core-compatibility .suite-case-toc"); await expect(sectionToc.locator(".suite-case-list a")).toHaveCount(2); const basicLink = coreIndex.getByRole("link", { name: "Basic response", exact: true }); const profile = page.locator("select[data-report-profile]"); const feature = page.locator("select[data-report-feature]"); + const crumbs = page.getByRole("navigation", { name: "Breadcrumb", exact: true }); + const returnToCampaign = crumbs.getByRole("link", { name: "navigation-fixture", exact: true }); await profile.selectOption("legacy-claude"); - const claudeResult = page.locator('[data-execution-id="core-compatibility.legacy-claude.local.message-marker"]'); + const readingPosition = await page.evaluate(() => window.scrollY); await basicLink.click(); - await expect(claudeResult.locator(".case-heading")).toBeInViewport(); + const claudeResult = page.locator('[data-execution-id="core-compatibility.legacy-claude.local.message-marker"]'); + await expect(claudeResult).toBeVisible(); + await expect(page.locator("[data-report-case]:visible")).toHaveCount(1); + await expect(page.getByRole("button", { name: "Legacy Claude · Isolated local", exact: true })).toHaveAttribute("aria-pressed", "true"); + await expect(page.locator(".report-filters")).toBeHidden(); + expect(new URL(page.url()).searchParams.get("view")).toBe("case"); + expect(new URL(page.url()).hash).toBe("#execution-core-compatibility.legacy-claude.local.message-marker"); + await returnToCampaign.click(); await expect(profile).toHaveValue("legacy-claude"); - await expect(page).toHaveURL(`${reportUrl}#execution-core-compatibility.legacy-claude.local.message-marker`); + await expect.poll(() => page.evaluate(() => window.scrollY)).toBe(readingPosition); - for (let click = 0; click < 2; click += 1) { - await feature.selectOption("questions-and-approvals"); - await basicLink.click(); - await expect(feature).toHaveValue(""); - await expect(profile).toHaveValue(""); - await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.local.message-marker"] .case-heading')).toBeInViewport(); - } + await feature.selectOption("questions-and-approvals"); + await basicLink.click(); + await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.local.message-marker"]')).toBeVisible(); + await expect(page.locator("[data-nav-configuration]:visible")).toHaveCount(3); + await crumbs.getByRole("link", { name: "Core Runner Compatibility", exact: true }).click(); + await expect(feature).toHaveValue(""); + await expect(profile).toHaveValue(""); + await expect(page.locator("[data-report-case]:visible")).toHaveCount(5); await sectionToc.getByRole("link", { name: "Ask mode question", exact: true }).click(); await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.local.ask-question"] .case-heading')).toBeInViewport(); - await sectionToc.getByRole("link", { name: "Back to index", exact: false }).click(); - await expect(coreIndex.locator("summary")).toBeInViewport(); + await sectionToc.getByRole("link", { name: "Back to overview", exact: false }).click(); + await expect(page.locator("#overview")).toBeInViewport(); await page.setViewportSize({ width: 390, height: 844 }); + await expect(coreIndex).toBeHidden(); + await page.getByRole("button", { name: "Open navigation", exact: true }).click(); await expect(basicLink).toBeVisible(); expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); await basicLink.click(); + await expect(coreIndex).toBeHidden(); + const daytona = page.getByRole("button", { name: "Legacy Codex · Daytona sandbox", exact: true }); + await daytona.click(); + await page.reload(); + await expect(daytona).toHaveAttribute("aria-pressed", "true"); + await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.daytona.message-marker"]')).toBeVisible(); + await expect(page.locator("[data-report-case]:visible")).toHaveCount(1); + await crumbs.getByRole("link", { name: "Core Runner Compatibility", exact: true }).click(); const backToTop = page.getByRole("link", { name: "Back to top", exact: true }); await expect(backToTop).toBeVisible(); expect(await backToTop.evaluate((element) => { @@ -306,28 +322,31 @@ test("folds the suite index and links each case to a matching configuration", as test("case status dots summarize matching results and open failures", async ({ page }) => { const ids = [...executionIds, "core-compatibility.legacy-codex.local.ask-question"]; const entries = executionIds.map((id, index) => resultFor(runnerExecutionById(id), index === 1 ? "failed" : "passed")); - const reportUrl = "http://runner-e2e.test/campaign/"; - await page.route(reportUrl, (route) => route.fulfill({ contentType: "text/html", body: navigationReport(ids, entries) })); - await page.goto(reportUrl); + await openReport(page, navigationReport(ids, entries)); const toc = page.locator("#suite-core-compatibility .suite-case-toc"); const basic = toc.getByRole("link", { name: "Basic response", exact: true }); const missing = toc.getByRole("link", { name: "Ask mode question", exact: true }); - const topLink = page.locator('.campaign-case-index [data-report-case-link="core-compatibility.message-marker"]'); + const topLink = page.locator('.report-suite-tree [data-report-case-link="core-compatibility.message-marker"]'); await expect(basic).toHaveAttribute("data-case-status", "failed"); await expect(topLink).toHaveAttribute("data-case-status", "failed"); - await expect(basic).toHaveAccessibleDescription("Failed · 2/3 passed · 1 failed — All configurations"); + await expect(basic).toHaveAccessibleDescription("failed · 2/3 configurations passed"); await expect(missing).toHaveAttribute("data-case-status", "missing"); - await expect(missing).toHaveAttribute("title", "Missing result · 0/1 passed · 1 missing — All configurations"); + await expect(missing).toHaveAttribute("title", "missing · 0/1 configurations passed"); const red = await basic.locator(".case-status-dot").evaluate((element) => getComputedStyle(element).backgroundColor); const profile = page.locator("select[data-report-profile]"); await profile.selectOption("legacy-claude"); await expect(basic).toHaveAttribute("data-case-status", "passed"); await expect(topLink).toHaveAttribute("data-case-status", "passed"); - await expect(basic).toHaveAccessibleDescription("Passed · 1/1 passed — Matching configurations"); + await expect(basic).toHaveAccessibleDescription("passed · 1/1 configurations passed"); const green = await basic.locator(".case-status-dot").evaluate((element) => getComputedStyle(element).backgroundColor); expect(green).not.toBe(red); - await expect(missing).toHaveAttribute("title", /none match current filters/); + await expect(missing).toHaveAttribute("title", /no matching configurations/); + await page.locator('[data-report-journey="core-compatibility.message-marker"]').getByRole("link", { name: "Inspect case ↗", exact: true }).click(); + await expect(page.locator('[data-execution-id="core-compatibility.legacy-claude.local.message-marker"]')).toBeVisible(); + await expect(page.locator("[data-report-case]:visible")).toHaveCount(1); + await expect(topLink).toHaveAttribute("data-case-status", "passed"); + await page.getByRole("navigation", { name: "Breadcrumb", exact: true }).getByRole("link", { name: "navigation-fixture", exact: true }).click(); await profile.selectOption("legacy-codex"); await expect(basic).toHaveAttribute("data-case-status", "failed"); @@ -335,6 +354,12 @@ test("case status dots summarize matching results and open failures", async ({ p await expect(basic).toHaveAttribute("data-case-status", "passed"); await page.locator("[data-report-reset]").click(); await basic.click(); - await expect(page).toHaveURL(`${reportUrl}#execution-core-compatibility.legacy-codex.daytona.message-marker`); + await expect(page).toHaveURL(`${reportUrl}?view=all#execution-core-compatibility.legacy-codex.daytona.message-marker`); await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.daytona.message-marker"] .case-heading')).toBeInViewport(); + await page.locator('[data-execution-id="core-compatibility.legacy-codex.local.message-marker"] .case-open').click(); + await expect(page.locator("[data-report-case]:visible")).toHaveCount(1); + await expect(topLink).toHaveAttribute("data-case-status", "failed"); + await expect(topLink).toHaveAccessibleDescription("failed · 2/3 configurations passed"); + await page.getByRole("button", { name: "Legacy Codex · Daytona sandbox", exact: true }).click(); + await expect(topLink).toHaveAttribute("data-case-status", "failed"); }); diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index f9ff0e2223b..f17b530e77b 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -778,15 +778,19 @@ captures remain failed evidence; qualifying a fix requires a new live attempt. Every completed local campaign also writes `tests/runner-e2e/results//dashboard.html`. The self-contained page -shows the complete profile/environment grid with screenshot thumbnails. +shows every selected execution with screenshot thumbnails, grouped by suite and case. The history homepage and campaign report render their campaign summaries as selectable HTML text, with suite links, pass counts, status, and timing. Campaign reports render only selected executions, including selected cases with missing evidence or startup failures. Search and feature/suite/profile filters -replace the full-catalog sticky index. -Each suite in the campaign report summary folds open to a list of its selected cases. -The same case links appear at the start of the suite, with a return to the top -index. Cases appear once across agent/environment configurations; links prefer +sit alongside a folding suite/case sidebar scoped to the current campaign. +Global Overview, Campaign history, and Needs attention pages have no suite tree. +All results is one continuous report with an overview and all execution previews. +Suite links jump into the report; sidebar case links open a focused case view +with a configuration picker. Clickable Campaign / Suite / Case breadcrumbs +return to the report and preserve its reading position. Suite contents also link +to execution previews, with a return to the overview and a bottom-right Back to top. +Cases appear once in the index across agent/environment configurations; links prefer a result matching the active filters and clear conflicting filters when needed to reveal their destination. Case links show a green status dot when all matching configurations passed, diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 28e8c0e0003..c9fe698d04d 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -1,3 +1,6 @@ +import { html, durationLabel, tokenLabel, llmCostLabel, usdLabel } from "./report-format.js"; +import { renderReportShell, reportNavigationStyles, reportShellScript } from "./report-navigation.js"; +import { reportBrowserScript } from "./report-browser.js"; import { isIncompleteFirstTaskResult, renderCaseOutcome } from "./case-outcome.js"; import { campaignSuiteSummaryStyles, renderCampaignSuiteSummary, renderSuiteCaseIndex } from "./campaign-summary.js"; import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js"; @@ -9,7 +12,6 @@ import { type ReportExecution, } from "./report-catalog.js"; import type { - RunnerE2EBillingSummary, RunnerE2ECampaign, RunnerE2EHistoryIndex, RunnerE2EResult, @@ -17,7 +19,6 @@ import type { } from "./types.js"; import { aggregateCampaignBilling, - billingCoverageLabel, summarizeExecutionBilling, } from "./billing.js"; @@ -46,36 +47,6 @@ interface ResolvedScreenshot { href: string; } -function html(value: unknown) { - return String(value ?? "") - .replaceAll("&", "&") - .replaceAll("<", "<") - .replaceAll(">", ">") - .replaceAll('"', """) - .replaceAll("'", "'"); -} - -function durationLabel(durationMs: number) { - if (durationMs < 1_000) return `${durationMs}ms`; - const seconds = Math.round(durationMs / 1_000); - if (seconds < 60) return `${seconds}s`; - return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; -} - -function tokenLabel(value: number, llm?: RunnerE2EBillingSummary["llm"]) { - const formatted = new Intl.NumberFormat("en-US").format(value); - return llm ? billingCoverageLabel(formatted, llm.runsWithTokenUsage, llm.runCount) : formatted; -} - -function llmCostLabel(value: number, llm: RunnerE2EBillingSummary["llm"]) { - return billingCoverageLabel(usdLabel(value), llm.runsWithReportedCost, llm.runCount); -} - -function usdLabel(value: number | null) { - if (value === null) return "Unknown"; - return `$${value.toFixed(value < 0.01 ? 6 : 4)}`; -} - function safeEvidenceHref(base: string | undefined, relative: string) { if (!base || /^(?:[a-z]+:|\/\/)/i.test(base)) return null; const cleanBase = base @@ -297,6 +268,7 @@ function renderCase( class="case case-${state}" data-execution-id="${html(execution.id)}" data-report-case + data-report-label="${html(execution.task.label)}" data-report-case-key="${html(`${execution.suite.id}.${execution.task.id}`)}" data-report-suite="${html(execution.suite.id)}" data-report-features="${html(journeyFeatureIds(execution).join(" "))}" @@ -306,7 +278,7 @@ function renderCase( data-report-search="${html(searchText)}" >
- ${html(execution.task.label)} + ${html(execution.task.label)} ${html(label)}
${features.length ? `
${features.map((feature) => @@ -346,230 +318,7 @@ function renderCase( `; } -function renderTrendChart(input: { - history: RunnerE2EHistoryIndex; - label: string; - value(campaign: RunnerE2EHistoryIndex["campaigns"][number]): number | null; - format(value: number): string; - include?(campaign: RunnerE2EHistoryIndex["campaigns"][number]): boolean; - fingerprint?(campaign: RunnerE2EHistoryIndex["campaigns"][number]): string; -}) { - const campaigns = input.history.campaigns - .filter(input.include ?? ((campaign) => campaign.complete)) - .slice(0, 20) - .reverse(); - if (campaigns.length === 0) { - return `
${html(input.label)}No complete campaigns
`; - } - const rawValues = campaigns.map(input.value); - const values = rawValues.filter((value): value is number => value !== null); - if (values.length !== rawValues.length) { - const latest = rawValues.at(-1); - return `
${html(input.label)}${latest == null ? "Unknown" : html(input.format(latest))}Unknown measurements are not plotted. Inspect campaign billing and reserved judge budgets.
`; - } - const maximum = Math.max(...values, 1); - const pointRows = values.map((value, index) => { - const x = - campaigns.length === 1 ? 50 : (index / (campaigns.length - 1)) * 100; - const y = 96 - (value / maximum) * 88; - return { - point: `${x.toFixed(2)},${y.toFixed(2)}`, - x, - y, - fingerprint: input.fingerprint?.(campaigns[index]!) ?? "stable", - }; - }); - const segments = pointRows.reduce>( - (groups, point) => { - const current = groups.at(-1); - if (!current || current.at(-1)?.fingerprint !== point.fingerprint) { - groups.push([point]); - } else { - current.push(point); - } - return groups; - }, - [], - ); - const definitionCount = new Set(pointRows.map((point) => point.fingerprint)) - .size; - const latest = values.at(-1) ?? 0; - return `
- ${html(input.label)} - ${html(input.format(latest))} - - ${segments.map((segment) => ``).join("")} - ${pointRows.map((point) => ``).join("")} - - ${campaigns.length} complete campaign${campaigns.length === 1 ? "" : "s"} · ${definitionCount} definition${definitionCount === 1 ? "" : "s"} -
`; -} - -function suiteSummaryFor( - campaign: RunnerE2EHistoryIndex["campaigns"][number], - suiteId: string, -) { - return campaign.suites.find((suite) => suite.suiteId === suiteId); -} - -function renderHistory(history: RunnerE2EHistoryIndex | undefined) { - if (!history || history.campaigns.length === 0) { - return `

History

Campaign trends

No historical campaigns have been published yet.

`; - } - const suiteIds = [ - ...new Set( - history.campaigns.flatMap((campaign) => - campaign.suites.map((suite) => suite.suiteId), - ), - ), - ]; - const charts = [ - renderTrendChart({ - history, - label: "Observed + estimated cost", - value: (campaign) => campaign.billing.observedAndEstimatedCostUsd, - format: usdLabel, - fingerprint: (campaign) => - campaign.suites - .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) - .sort() - .join("|"), - }), - renderTrendChart({ - history, - label: "Total tokens", - value: (campaign) => campaign.billing.llm.totalTokens, - format: tokenLabel, - fingerprint: (campaign) => - campaign.suites - .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) - .sort() - .join("|"), - }), - renderTrendChart({ - history, - label: "Agent execution time", - value: (campaign) => campaign.billing.agentRunDurationMs, - format: durationLabel, - fingerprint: (campaign) => - campaign.suites - .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) - .sort() - .join("|"), - }), - renderTrendChart({ - history, - label: "Daytona lease time", - value: (campaign) => campaign.billing.leaseDurationMs, - format: durationLabel, - fingerprint: (campaign) => - campaign.suites - .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) - .sort() - .join("|"), - }), - renderTrendChart({ - history, - label: "Pass rate", - value: (campaign) => - campaign.selected > 0 ? (campaign.passed / campaign.selected) * 100 : 0, - format: (value) => `${value.toFixed(1)}%`, - fingerprint: (campaign) => - campaign.suites - .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) - .sort() - .join("|"), - }), - ].join(""); - const suiteCharts = suiteIds - .map((suiteId) => { - const include = (campaign: RunnerE2EHistoryIndex["campaigns"][number]) => - suiteSummaryFor(campaign, suiteId)?.complete === true; - const value = ( - campaign: RunnerE2EHistoryIndex["campaigns"][number], - metric: "cost" | "tokens" | "agent" | "lease" | "passRate", - ) => { - const suite = suiteSummaryFor(campaign, suiteId); - if (!suite) return 0; - if (metric === "cost") return suite.billing.observedAndEstimatedCostUsd; - if (metric === "tokens") return suite.billing.llm.totalTokens; - if (metric === "agent") return suite.billing.agentRunDurationMs; - if (metric === "lease") return suite.billing.leaseDurationMs; - return suite.selected > 0 ? (suite.passed / suite.selected) * 100 : 0; - }; - const fingerprint = ( - campaign: RunnerE2EHistoryIndex["campaigns"][number], - ) => suiteSummaryFor(campaign, suiteId)?.suiteDefinitionHash ?? "unknown"; - return `
-

${html(suiteId)}

Complete suite selections only; lines break at definition changes.
-
- ${renderTrendChart({ history, label: "Suite cost", value: (campaign) => value(campaign, "cost"), format: usdLabel, include, fingerprint })} - ${renderTrendChart({ history, label: "Suite tokens", value: (campaign) => value(campaign, "tokens"), format: tokenLabel, include, fingerprint })} - ${renderTrendChart({ history, label: "Suite agent time", value: (campaign) => value(campaign, "agent"), format: durationLabel, include, fingerprint })} - ${renderTrendChart({ history, label: "Suite lease time", value: (campaign) => value(campaign, "lease"), format: durationLabel, include, fingerprint })} - ${renderTrendChart({ history, label: "Suite pass rate", value: (campaign) => value(campaign, "passRate"), format: (metric) => `${metric.toFixed(1)}%`, include, fingerprint })} -
-
`; - }) - .join(""); - const rows = history.campaigns - .map((campaign) => { - const status = campaign.failed > 0 ? "failed" : (campaign.incomplete ?? 0) > 0 ? "incomplete" : "passed"; - const sha = campaign.source.sha; - const searchable = [ - campaign.campaignId, - sha, - campaign.source.ref, - ...campaign.executions.flatMap((execution) => [ - execution.suiteId, - execution.profileId, - execution.model, - execution.environmentId, - execution.caseId, - execution.status, - ]), - ] - .filter(Boolean) - .join(" ") - .toLowerCase(); - return ` - ${html(campaign.campaignId)}${html(new Date(campaign.generatedAt).toLocaleString("en-US", { timeZone: "UTC" }))} UTC - ${sha ? `${html(sha.slice(0, 10))}` : "Unknown"}${html(campaign.source.ref ?? "unknown ref")} - ${status}${campaign.passed}/${campaign.selected} passed${campaign.incomplete ? ` · ${campaign.incomplete} incomplete` : ""} · ${campaign.complete ? "complete" : "partial"} - ${html(tokenLabel(campaign.billing.llm.inputTokens, campaign.billing.llm))} / ${html(tokenLabel(campaign.billing.llm.outputTokens, campaign.billing.llm))}input / output · ${html(tokenLabel(campaign.billing.llm.cachedInputTokens, campaign.billing.llm))} cached - ${html(llmCostLabel(campaign.billing.reportedLlmCostUsd, campaign.billing.llm))}${html(usdLabel(campaign.billing.estimatedRuntimeCostUsd))} runtime estimate - ${html(durationLabel(campaign.billing.agentRunDurationMs))}${html(durationLabel(campaign.billing.leaseDurationMs))} lease - `; - }) - .join(""); - const latest = history.campaigns.find( - (campaign) => campaign.campaignId === history.latestCampaignId, - ); - const latestGreen = history.campaigns.find( - (campaign) => campaign.campaignId === history.latestGreenCampaignId, - ); - return `
-

History

Campaign trends

Complete campaigns are compared by default. Partial smoke runs remain searchable and are labeled explicitly.

- -
${charts}
- ${suiteCharts} -
- - - - - - -
-
${rows}
CampaignPaperclip SHAResultTokensCostExecution
- -
`; -} - -function renderSuiteMatrix(input: { +function renderSuiteResults(input: { suiteCatalog: readonly ReportExecution[]; expected: ReadonlySet; entryById: ReadonlyMap; @@ -577,52 +326,6 @@ function renderSuiteMatrix(input: { }) { const suite = input.suiteCatalog[0]?.suite; if (!suite) return ""; - const profiles = [ - ...new Map( - input.suiteCatalog.map((execution) => [ - execution.profile.id, - execution.profile, - ]), - ).values(), - ]; - const environments = [ - ...new Map( - input.suiteCatalog.map((execution) => [ - execution.environment.id, - execution.environment, - ]), - ).values(), - ]; - const rows = profiles - .map((profile, profileIndex) => { - const columns = environments - .map((environment) => { - const executions = input.suiteCatalog.filter( - (execution) => - execution.profile.id === profile.id && - execution.environment.id === environment.id, - ); - return ` -
${html(environment.label)}${html(environment.provider)} · ${html(environment.expectedExecutionTarget.kind)}
-
${executions.map((execution) => renderCase(execution, input.expected, input.entryById)).join("")}
- `; - }) - .join(""); - return `
- - ${html(profile.label)}${html(profile.generation)}${html(profile.provider)} · ${html(suite.id === "first-task" ? "Production onboarding default" : profile.model)} -
${columns}`; - }) - .join(""); - const environmentHeaders = environments - .map( - (environment) => - `${html(environment.label)}${html(environment.provider)} · ${html(environment.expectedExecutionTarget.kind)}`, - ) - .join(""); - const selected = input.suiteCatalog.filter((execution) => - input.expected.has(execution.id), - ).length; const summary = input.summary; const summaryHtml = summary ? `
@@ -633,19 +336,27 @@ function renderSuiteMatrix(input: {
Execution${summary.executed}/${summary.selected}${summary.retries} retries · cleanup ${summary.cleanupPassed ? "passed" : "failed"}
` : ""; - return `
+ const journeys = [...new Map(input.suiteCatalog.map(row => [row.task.id, row.task])).values()]; + return `

Test suite

${html(suite.label)}

${html(suite.description)}

${summaryHtml} -
Configuration matrix${profiles.length} profiles · ${environments.length} environments · ${selected} selected
-
${environments.map(() => '').join("")}${environmentHeaders}${rows}
Agent profile
+ ${journeys.map(task => { + const key = `${suite.id}.${task.id}`; + const executions = input.suiteCatalog.filter(row => row.task.id === task.id); + const href = `?view=case&case=${encodeURIComponent(key)}`; + return `
+

${html(task.label)}

Inspect case ↗
+ +
${executions.map(row => `
${html(row.profile.label)}${html(row.environment.label)}${html(row.profile.provider)} · ${html(row.profile.model)}
${renderCase(row, input.expected, input.entryById)}
`).join("")}
+
`; + }).join("")}
`; } - export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { const expected = new Set(input.expected); const catalog = discoverReportCatalog({ @@ -682,7 +393,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { ]; const suiteSections = suites .map((suite) => - renderSuiteMatrix({ + renderSuiteResults({ suiteCatalog: catalog.filter( (execution) => execution.suite.id === suite.id, ), @@ -694,7 +405,6 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { }), ) .join(""); - const historySection = renderHistory(input.history); const filterProfiles = [ ...new Map( catalog.map((execution) => [execution.profile.id, execution.profile]), @@ -925,40 +635,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .suite-case-toc-heading { display: flex; align-items: baseline; justify-content: space-between; gap: 16px; margin-bottom: 8px; } .suite-case-toc-heading h3 { margin: 0; font-size: 13px; font-weight: 600; } .suite-case-toc-heading > a { color: var(--muted-foreground); font-size: 11px; } - .history-pointers { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 12px; margin-bottom: 16px; } - .history-pointers a { padding: 14px 16px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--card); text-decoration: none; } - .history-pointers span, .history-pointers strong { display: block; } - .history-pointers span { color: var(--muted-foreground); font-size: 10px; letter-spacing: .06em; text-transform: uppercase; } - .history-pointers strong { margin-top: 4px; font: 550 12px/1.4 var(--font-mono); overflow-wrap: anywhere; } - .trend-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; } - .trend-card { min-width: 0; padding: 14px 16px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--card); } - .trend-card > span, .trend-card > strong, .trend-card > small { display: block; } - .trend-card > span, .trend-card > small { color: var(--muted-foreground); font-size: 10px; } - .trend-card > span { letter-spacing: .06em; text-transform: uppercase; } - .trend-card > strong { margin-top: 4px; font: 550 17px/1.3 var(--font-mono); } - .trend-card svg { width: 100%; height: 72px; margin: 10px 0 4px; overflow: visible; } - .trend-card polyline { stroke: var(--foreground); stroke-width: 1.6; } - .trend-card circle { fill: var(--background); stroke: var(--foreground); stroke-width: 1; } - .suite-trends { margin-top: 28px; padding-top: 20px; border-top: 1px solid var(--border); } - .suite-trends-heading { display: flex; align-items: baseline; justify-content: space-between; gap: 16px; margin-bottom: 12px; } - .suite-trends-heading h3 { margin: 0; font: 600 14px/1.4 var(--font-mono); } - .suite-trends-heading span { color: var(--muted-foreground); font-size: 10px; } - .history-filters { display: grid; grid-template-columns: minmax(220px, 1fr) repeat(4, minmax(130px, 180px)) auto; align-items: end; gap: 12px; margin: 24px 0 12px; } - .history-filters label { display: grid; gap: 5px; color: var(--muted-foreground); font-size: 10px; letter-spacing: .05em; text-transform: uppercase; } - .history-filters input, .history-filters select { width: 100%; min-height: 38px; padding: 7px 9px; border: 1px solid var(--border-strong); border-radius: calc(var(--radius) * .8); background: var(--background); color: var(--foreground); } - .history-filters .history-checkbox { display: flex; align-items: center; min-height: 38px; padding-bottom: 7px; white-space: nowrap; } - .history-checkbox input { width: 16px; min-height: 16px; } - .history-table-wrap { overflow-x: auto; border-top: 1px solid var(--border); border-bottom: 1px solid var(--border); } - .history-table { width: 100%; min-width: 1020px; border-collapse: collapse; } - .history-table th, .history-table td { padding: 12px; border-bottom: 1px solid var(--border); text-align: left; vertical-align: top; } - .history-table tr:last-child td { border-bottom: 0; } - .history-table th { background: var(--raised); color: var(--muted-foreground); font-size: 10px; letter-spacing: .05em; text-transform: uppercase; } - .history-table td small { display: block; margin-top: 3px; color: var(--muted-foreground); font-size: 10px; } - .history-table .status { margin-bottom: 2px; } - .history-passed { border-color: var(--pass-border); background: var(--pass-bg); color: var(--pass-text); } - .history-incomplete { border-color: var(--missing-border); background: var(--missing-bg); color: var(--missing-text); } - .history-failed { border-color: var(--fail-border); background: var(--fail-bg); color: var(--fail-text); } - .history-empty { padding: 24px 0; color: var(--muted-foreground); text-align: center; } + .case-context > summary, .usage > summary { width: fit-content; cursor: pointer; color: var(--foreground); font-size: 12px; font-weight: 600; } .case-diagnostics { margin-top: 14px; padding-top: 12px; border-top: 1px solid var(--border); } .case-diagnostics > summary { display: flex; align-items: baseline; flex-wrap: wrap; gap: 4px 8px; width: 100%; list-style: none; } @@ -1126,22 +803,20 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .table-wrap { max-height: none; overflow: visible; } .matrix thead th, .profile-cell, .profile-sticky { position: static; } } + ${reportNavigationStyles} -
- - - Paperclip - - Quality engineering · Runner acceptance -
+ ${renderReportShell({ campaignId: input.campaign?.campaignId, catalog, history: input.history })}
+ +

Full-stack acceptance campaign

${html(input.title)}

-

A browser-verified matrix of runner profiles, execution environments, and deterministic task contracts. Declared PNG screenshots and normalized results are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.

+

Results across runner profiles and execution environments. Declared PNG screenshots and normalized results are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.

+ ${input.campaign ? `

${html(input.generatedAt)}${input.campaign.source.sha ? `${html(input.campaign.source.sha.slice(0,8))}` : ""}${input.campaign.source.workflowRunUrl ? `Source run ↗` : ""}

` : ""}
@@ -1153,7 +828,8 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
- ${renderCampaignSuiteSummary(input.campaign?.suites ?? [], catalog, "", true)} + ${renderCampaignSuiteSummary(input.campaign?.suites ?? [], catalog, "", false)} +
Campaign usage and spend
${html(tokenLabel(campaignBilling.llm.inputTokens, campaignBilling.llm))}Input tokens
${html(tokenLabel(campaignBilling.llm.outputTokens, campaignBilling.llm))}Output tokens
@@ -1166,9 +842,10 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
${html(durationLabel(campaignBilling.leaseDurationMs))}Daytona lease time
${campaignBilling.llm.runsWithReportedCost}/${campaignBilling.llm.runCount}Runs provider-priced

Model spend is the provider-reported subtotal; unpriced or unavailable runs are excluded, never counted as free. Daytona runtime is a public-list-price estimate from captured lease time and pinned resources, before credits, discounts, storage allowance, or invoice adjustments. Local execution has no external runtime meter.

+
- - ${renderJourneyFeatureCoverage(catalog, entryById)} +

All results

+
${renderJourneyFeatureCoverage(catalog, entryById)}
@@ -1180,9 +857,9 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {

${suiteSections} - ${historySection}
Generated ${html(input.generatedAt)}${catalog.length} selected executions · Declared screenshots and normalized results published
+
@@ -1225,8 +902,9 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { if (event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey) return; event.preventDefault(); window.scrollTo({ top: 0, behavior: "instant" }); - if (location.hash !== "#top") history.pushState(null, "", "#top"); - document.querySelector(".report-header h1").focus({ preventScroll: true }); + const hash = document.querySelector('.report-shell').dataset.view === 'case' ? location.hash : ''; + history.replaceState({ ...history.state, reportScroll: 0 }, "", location.pathname + location.search + hash); + document.querySelector("[data-case-page-heading]:not([hidden]) h1, .report-header h1").focus({ preventScroll: true }); update(); }); update(); @@ -1330,170 +1008,8 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { stage.addEventListener("pointercancel", () => { pointerStartX = null; }); })(); - - + + `; diff --git a/tests/runner-e2e/history-index.ts b/tests/runner-e2e/history-index.ts index cf030f2514d..5c72cb9dde3 100644 --- a/tests/runner-e2e/history-index.ts +++ b/tests/runner-e2e/history-index.ts @@ -1,3 +1,5 @@ +import { renderReportHistory, reportHistoryStyles, reportHistoryScript } from "./report-history.js"; +import { renderReportShell, reportNavigationStyles, reportShellScript } from "./report-navigation.js"; import { runnerMatrix } from "./catalog.js"; import { campaignSuiteSummaryStyles, renderCampaignSuiteSummary } from "./campaign-summary.js"; import type { @@ -70,34 +72,26 @@ function campaignRow(campaign: RunnerE2EHistoryCampaign) { ) .join(""); const billing = campaign.billing; - return ` + return ` ${html(campaign.campaignId)} ${html(date(campaign.generatedAt))} UTC - + ${status} - ${campaign.complete ? "Complete campaign" : "Partial campaign"}${campaign.retries > 0 ? ` · ${campaign.retries} retries` : ""} + ${campaign.passed}/${campaign.selected} passed · ${campaign.executed} executed ${sourceCell(campaign)} -
${suites}
- - ${campaign.passed}/${campaign.selected} passed - ${campaign.executed} executed · ${campaign.failed} failed${campaign.incomplete ? ` · ${campaign.incomplete} incomplete` : ""} + Open report → +
Suites & billing +
${suites}
+

${campaign.complete ? "Complete campaign" : "Partial campaign"} · ${campaign.retries} retries · cleanup ${campaign.cleanupPassed ? "passed" : "failed"}

+

${campaign.failed} failed${campaign.incomplete ? ` · ${campaign.incomplete} incomplete` : ""}

+

${html(number(billing.llm.totalTokens))} tokens · ${html(number(billing.llm.inputTokens))} in · ${html(number(billing.llm.outputTokens))} out · ${html(number(billing.llm.cachedInputTokens))} cached

+

${html(usd(billing.observedAndEstimatedCostUsd))} recorded cost · ${html(usd(billing.reportedLlmCostUsd))} LLM · ${html(usd(billing.estimatedRuntimeCostUsd))} runtime

+

${html(duration(billing.agentRunDurationMs))} agent · ${html(duration(billing.leaseDurationMs))} Daytona lease

+
- - ${html(number(billing.llm.totalTokens))} - ${html(number(billing.llm.inputTokens))} in · ${html(number(billing.llm.outputTokens))} out · ${html(number(billing.llm.cachedInputTokens))} cached - - - ${html(usd(billing.observedAndEstimatedCostUsd))} - ${html(usd(billing.reportedLlmCostUsd))} LLM · ${html(usd(billing.estimatedRuntimeCostUsd))} runtime - - - ${html(duration(billing.agentRunDurationMs))} - ${html(duration(billing.leaseDurationMs))} Daytona lease - - Open report → `; } @@ -123,7 +117,7 @@ export function renderRunnerHistoryIndex( const rows = campaigns.length > 0 ? campaigns.map(campaignRow).join("") - : `No campaigns have been published yet.`; + : `No campaigns have been published yet.`; return ` @@ -195,22 +189,36 @@ export function renderRunnerHistoryIndex( .open-cell { display:block; padding-top:14px; text-align:left; } .open-cell::before { display:none; } } @media (max-width:520px) { .summary { grid-template-columns:1fr; } .metric { border-right:0; } footer { flex-direction:column; } } + :root { --muted-foreground:var(--muted); --card:var(--raised); --border-strong:var(--border); --font-sans:var(--font); --font-mono:var(--mono); --radius:8px; --pass-text:var(--pass); --pass-border:var(--pass); --fail-text:var(--fail); --fail-border:var(--fail); --missing-text:#b45309; --missing-bg:#fef3c7; --missing-border:#f59e0b; } + .index-search { display:flex; justify-content:space-between; align-items:center; gap:20px; margin:28px 0 14px; } + .index-search h2 { margin:0; font-size:15px; } + .index-search input { width:300px; max-width:65%; min-height:36px; padding:8px 10px; background:var(--background); border:1px solid var(--border); border-radius:5px; color:var(--foreground); font:inherit; } + .index-result { color:var(--muted); font-size:11px; } + .report-surface h1 { font-size:32px; } + .report-surface .lede { font-size:12px; } + .report-surface .table-wrap { overflow-x:auto; } + .report-surface table { table-layout:fixed; } + .report-surface th:first-child { width:29%; } + .report-surface th:nth-child(2) { width:20%; } + .report-surface th:nth-child(3) { width:28%; } + .report-surface td { overflow-wrap:anywhere; } + .campaign-row-details { margin-top:8px; color:var(--muted); font-size:11px; text-align:left; font-weight:400; } + .campaign-row-details summary { cursor:pointer; text-align:right; } + .campaign-row-details p { margin:10px 0; } + .campaign-row-details .suite-list { white-space:normal; } + ${reportNavigationStyles} + ${reportHistoryStyles} + @media(max-width:860px) { .report-surface table { min-width:0; } } -
- - - Paperclip - - Quality engineering · Runner acceptance -
+ ${renderReportShell({ history })}

Historical test reporting

-

Runner E2E campaigns

-

Each row is one workflow campaign against a Paperclip revision. Open a report for its configuration matrices, matchers, per-test billing, declared screenshots, and normalized results. Additional diagnostic evidence remains in access-controlled workflow artifacts.

+

Runner E2E campaigns

+

${campaigns.length} retained campaigns. Open a report for test outcomes, per-test billing, declared screenshots, and normalized results.

${campaigns.length}Campaigns
@@ -218,6 +226,7 @@ export function renderRunnerHistoryIndex(
${html(usd(totalCost))}Recorded cost
+
${latest ? `

Latest campaign

${html(latest.campaignId)}
@@ -231,14 +240,49 @@ export function renderRunnerHistoryIndex( ${latest ? `Latest run · ${html(latest.campaignId)}` : ""} ${latestGreen ? `Latest complete green · ${html(latestGreen.campaignId)}` : ""} +
+

Recent campaigns

+

- + ${rows}
CampaignStatusSourceSuitesTestsTokensCostAgent time
CampaignResultSource
+
Updated ${html(date(history.updatedAt))} UTCImmutable campaign reports · Declared screenshots and normalized results
-
+
+ `; } diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts index c5821f91d34..c60278769d0 100644 --- a/tests/runner-e2e/history.test.ts +++ b/tests/runner-e2e/history.test.ts @@ -137,7 +137,7 @@ describe("runner E2E campaign history", () => { expect(history.latestGreenCampaignId).toBeNull(); expect(history.latestGreenBySuite["first-task"]).toBeUndefined(); expect(renderRunnerHistoryIndex(history)).toContain("1 incomplete"); - expect(renderRunnerE2EDashboard({ title: "History", generatedAt: incomplete.finishedAt, expected: [], catalog: [], entries: [], history })).toContain('data-history-status="incomplete"'); + expect(renderRunnerHistoryIndex(history)).toContain('data-history-status="incomplete"'); }); it("shows unknown campaign costs without plotting zero dollars", () => { @@ -147,9 +147,9 @@ describe("runner E2E campaign history", () => { const record = { ...campaignHistoryRecord(campaign, "https://example.test"), complete: true }; const history = mergeRunnerHistory(null, record); expect(renderRunnerHistoryIndex(history)).toContain("Unknown"); - const dashboard = renderRunnerE2EDashboard({ title: "History", generatedAt: campaign.generatedAt, expected: [], catalog: [], entries: [], history }); - expect(dashboard).toContain("Unknown measurements are not plotted"); - expect(dashboard).not.toContain("NaN"); + const index = renderRunnerHistoryIndex(history); + expect(index).toContain("Unknown measurements are not plotted"); + expect(index).not.toContain("NaN"); }); it("does not turn a clean-evidence behavior failure into a pass when regenerating", async () => { @@ -274,22 +274,23 @@ describe("runner E2E campaign history", () => { errors: campaignResult.status === "passed" ? [] : ["failed"], })), }); - expect(dashboard).toContain("Campaign trends"); - expect(dashboard).toContain("data-history-from"); - expect(dashboard).toContain("data-history-through"); - expect(dashboard).toContain( + expect(dashboard).not.toContain('class="history-section"'); + const index = renderRunnerHistoryIndex(history); + expect(index).toContain("Campaign trends"); + expect(index).toContain("data-history-from"); + expect(index).toContain("data-history-through"); + expect(index).toContain( 'data-history-suite-trends="core-compatibility"', ); - expect(dashboard).toContain( + expect(index).toContain( 'data-history-suite-trends="openrouter-model-breadth"', ); - expect(dashboard).toContain( + expect(index).toContain( 'data-history-suite-trends="local-session-integrity"', ); - expect(dashboard).toContain("Suite pass rate"); - expect(dashboard).toContain("lines break at definition changes"); - expect(dashboard).toContain("cleanup passed"); - const index = renderRunnerHistoryIndex(history); + expect(index).toContain("Suite pass rate"); + expect(index).toContain("lines break at definition changes"); + expect(index).toContain("cleanup passed"); expect(index).toContain("Runner E2E campaigns"); expect(index).toContain("complete-green"); expect(index).toContain("complete-red"); diff --git a/tests/runner-e2e/report-brand.ts b/tests/runner-e2e/report-brand.ts new file mode 100644 index 00000000000..bd3988f1f08 --- /dev/null +++ b/tests/runner-e2e/report-brand.ts @@ -0,0 +1,12 @@ +export const paperclipReportLogo = ``; diff --git a/tests/runner-e2e/report-browser.ts b/tests/runner-e2e/report-browser.ts new file mode 100644 index 00000000000..d8dd0468e29 --- /dev/null +++ b/tests/runner-e2e/report-browser.ts @@ -0,0 +1,176 @@ +export const reportBrowserScript = String.raw` +(() => { + const shell = document.querySelector('[data-report-scope="campaign"]'); + if (!shell) return; + const cards = [...document.querySelectorAll('[data-report-case]')]; + const sections = [...document.querySelectorAll('[data-report-suite-section]')]; + const journeys = [...document.querySelectorAll('[data-report-journey]')]; + const controls = ['query','feature','profile','environment','suite','status'].map(name => document.querySelector('[data-report-' + name + ']')); + const [query,feature,profile,environment,suite,status] = controls; + const paramNames = ['q','feature','profile','environment','suite','status']; + const byKey = new Map(); + cards.forEach(card => { const key = card.dataset.reportCaseKey; if (!byKey.has(key)) byKey.set(key, []); byKey.get(key).push(card); }); + const crumbs = document.querySelector('[data-nav-breadcrumbs]'); + const historyHref = crumbs.querySelector('a').getAttribute('href'); + const heading = document.querySelector('[data-case-page-heading]'); + const result = document.querySelector('[data-report-result]'); + let view = 'all', caseKey = '', configuration = '', lastReport = null, restoring = false; + history.scrollRestoration = 'manual'; + const normalClick = event => event.button === 0 && !event.metaKey && !event.ctrlKey && !event.shiftKey && !event.altKey; + const savePosition = () => { if (!restoring) history.replaceState({ ...history.state, reportScroll: window.scrollY }, '', location.href); }; + const severity = rows => rows.some(row => row.dataset.reportStatus === 'failed') ? 'failed' : rows.some(row => row.dataset.reportStatus === 'incomplete') ? 'incomplete' : rows.some(row => row.dataset.reportStatus === 'missing') ? 'missing' : 'passed'; + const clearFilters = () => controls.forEach(control => { control.value = ''; }); + const routeUrl = (nextView, key = '', hash = '') => { + const url = new URL(location.href); + url.searchParams.set('view', nextView); + if (key) url.searchParams.set('case', key); else url.searchParams.delete('case'); + controls.forEach((control, i) => { if(control.value) url.searchParams.set(paramNames[i], control.value); else url.searchParams.delete(paramNames[i]); }); + url.hash = hash; + return url.pathname + url.search + url.hash; + }; + const readRoute = () => { + const params = new URLSearchParams(location.search); + view = ['all','attention','case'].includes(params.get('view')) ? params.get('view') : 'all'; + caseKey = params.get('case') || ''; + configuration = location.hash.startsWith('#execution-') ? decodeURIComponent(location.hash.slice(11)) : ''; + controls.forEach((control,i) => { control.value = params.get(paramNames[i]) || ''; }); + if(location.hash === '#history') { location.replace(historyHref); return false; } + return true; + }; + const appendCrumb = (label, href, attributes = {}) => { + if(crumbs.childNodes.length) { const separator = document.createElement('span'); separator.setAttribute('aria-hidden','true'); separator.textContent = '/'; crumbs.append(separator); } + const node = document.createElement(href ? 'a' : 'span'); node.textContent = label; + if(href) node.setAttribute('href', href); else node.setAttribute('aria-current','page'); + Object.entries(attributes).forEach(([name,value]) => node.setAttribute(name,value)); crumbs.append(node); + }; + const setBreadcrumbs = selected => { + crumbs.replaceChildren(); + appendCrumb('Campaigns', historyHref); + appendCrumb(shell.dataset.campaignLabel, lastReport?.url || routeUrl('all'), {'data-nav-return':''}); + if(view === 'case' && selected) { + const section = selected.closest('[data-report-suite-section]'); + appendCrumb(section.dataset.suiteLabel, routeUrl('all','','#' + section.id), {'data-nav-suite-link':selected.dataset.reportSuite}); + appendCrumb(selected.dataset.reportLabel); + } else appendCrumb(view === 'attention' ? 'Needs attention' : view === 'case' ? 'Case unavailable' : 'All results'); + }; + const apply = () => { + const search = query.value.trim().toLowerCase(); + const matchesFilters = card => + (!search || card.dataset.reportSearch.includes(search)) && + (!feature.value || (feature.value === '__unmapped__' ? !card.dataset.reportFeatures : card.dataset.reportFeatures.split(' ').includes(feature.value))) && + (!profile.value || card.dataset.reportProfile === profile.value) && + (!environment.value || card.dataset.reportEnvironment === environment.value) && + (!suite.value || card.dataset.reportSuite === suite.value) && + (!status.value || card.dataset.reportStatus === status.value); + const variants = byKey.get(caseKey) || []; + const matchingVariants = variants.filter(matchesFilters); + const candidates = matchingVariants.length ? matchingVariants : variants; + let selected = variants.find(row => row.dataset.executionId === configuration); + if(!selected) selected = candidates.find(row => row.dataset.reportStatus === severity(candidates)) || candidates[0]; + configuration = selected?.dataset.executionId || ''; + let visible = 0; + cards.forEach(card => { + const matches = view === 'case' ? card === selected : + (view !== 'attention' || card.dataset.reportStatus !== 'passed') && matchesFilters(card); + card.hidden = !matches; + card.closest('.execution-preview').hidden = !matches; + if(matches) visible++; + }); + journeys.forEach(journey => { + journey.hidden = !journey.querySelector('[data-report-case]:not([hidden])'); + journey.querySelector('.case-config-options').hidden = view !== 'case'; + }); + sections.forEach(section => { section.hidden = !section.querySelector('[data-report-case]:not([hidden])'); }); + document.querySelectorAll('[data-nav-configuration]').forEach(button => { + const card = document.getElementById('execution-' + button.dataset.navConfiguration); + button.setAttribute('aria-pressed', String(card === selected)); + button.dataset.caseStatus = card.dataset.reportStatus; + button.setAttribute('aria-description', card.dataset.reportStatus); + }); + document.querySelectorAll('[data-report-case-link]').forEach(link => { + const variants = byKey.get(link.dataset.reportCaseLink) || []; + const matching = variants.filter(matchesFilters); + const rows = matching.length ? matching : variants; + if(!rows.length) return; + const state = severity(rows), target = rows.find(row => row.dataset.reportStatus === state) || rows[0]; + const passed = rows.filter(row => row.dataset.reportStatus === 'passed').length; + const description = state.replaceAll('-', ' ') + ' · ' + passed + '/' + rows.length + ' configurations passed' + (matching.length ? '' : ' · no matching configurations'); + link.dataset.caseStatus = state; link.title = description; link.setAttribute('aria-description', description); + link.setAttribute('href', link.hasAttribute('data-nav-case') ? routeUrl('case', link.dataset.navCase, '#execution-' + target.dataset.executionId) : '#execution-' + target.dataset.executionId); + }); + shell.dataset.view = view; + document.querySelector('[data-campaign-overview]').hidden = view !== 'all'; + document.querySelector('[data-feature-coverage]').hidden = view === 'case'; + document.querySelector('.report-filters').hidden = view === 'case'; + document.querySelector('.results-heading').hidden = view === 'case'; + document.querySelector('[data-results-title]').textContent = view === 'attention' ? 'Needs attention' : 'All results'; + document.querySelector('[data-results-description]').textContent = view === 'attention' ? 'Failed, incomplete, and missing executions in this campaign' : ''; + heading.hidden = view !== 'case'; + document.querySelector('[data-case-page-title]').textContent = selected?.dataset.reportLabel || 'This case was not selected in this campaign'; + document.querySelector('[data-nav-return]').setAttribute('href', lastReport?.url || routeUrl('all')); + document.querySelectorAll('[data-nav-view]').forEach(link => { if(link.dataset.navView === view) link.setAttribute('aria-current','page'); else link.removeAttribute('aria-current'); }); + document.querySelectorAll('[data-nav-suite-link]').forEach(link => link.setAttribute('href',routeUrl('all','','#suite-' + link.dataset.navSuiteLink))); + document.querySelectorAll('[data-nav-case]').forEach(link => { if(view === 'case' && link.dataset.navCase === caseKey) link.setAttribute('aria-current','page'); else link.removeAttribute('aria-current'); }); + const selectedSuite = selected?.dataset.reportSuite; + if(view === 'case') document.querySelectorAll('[data-nav-suite]').forEach(details => { if(details.dataset.navSuite === selectedSuite) details.open = true; }); + result.textContent = visible + ' of ' + cards.length + ' executions shown' + (!visible ? ' — No executions match this selection.' : ''); + const images = cards.reduce((n,card) => n + (card.hidden ? 0 : card.querySelectorAll('[data-gallery-item]').length),0); + const launch = document.querySelector('[data-gallery-open]'); + launch.disabled = images === 0; launch.textContent = images ? 'View gallery · ' + images : 'No visual evidence in selection'; + setBreadcrumbs(selected); + measureHeader(); + }; + const measureHeader = () => document.documentElement.style.setProperty('--navigation-sticky-offset',document.querySelector('.report-topbar').offsetHeight + 'px'); + window.addEventListener('resize',measureHeader); + const navigate = (url, scroll) => { + savePosition(); history.pushState({reportScroll:scroll || 0},'',url); + readRoute(); restoring = true; apply(); + if(typeof scroll === 'number') window.scrollTo({top:scroll,behavior:'instant'}); + else if(location.hash) revealTarget(location.hash); + else window.scrollTo({top:0,behavior:'instant'}); + restoring = false; + }; + const revealTarget = hash => { + const id = decodeURIComponent(hash.slice(1)); + const target = document.getElementById(id); + if(!target || !target.matches('.suite-section,[data-report-case],.report-journey,#overview,#results,#top')) return; + if(target.hidden || target.closest('[hidden]')) { + clearFilters(); view = 'all'; caseKey = ''; apply(); + history.replaceState(history.state,'',routeUrl('all','',hash)); + } + target.scrollIntoView({block:'start',behavior:'instant'}); + }; + document.addEventListener('click', event => { + if(event.defaultPrevented || !normalClick(event)) return; + const link = event.target.closest('a'); + if(link?.hasAttribute('data-nav-return')) { + event.preventDefault(); navigate(lastReport?.url || routeUrl('all'),lastReport?.scroll || 0); return; + } + if(link?.hasAttribute('data-nav-case')) { + event.preventDefault(); + if(view !== 'case') lastReport = {url:location.pathname + location.search + location.hash,scroll:window.scrollY}; + const hash = new URL(link.href).hash; + navigate(routeUrl('case',link.dataset.navCase,hash),0); return; + } + if(link?.hasAttribute('data-nav-view')) { event.preventDefault(); navigate(routeUrl(link.dataset.navView)); return; } + if(link?.hasAttribute('data-nav-suite-link')) { + event.preventDefault(); const id = link.dataset.navSuiteLink; + document.querySelectorAll('[data-nav-suite]').forEach(details => {if(details.dataset.navSuite === id)details.open = true;}); + navigate(routeUrl('all','','#suite-' + id)); return; + } + if(link?.getAttribute('href')?.startsWith('#') && /^#(suite-|execution-|journey-|overview|results)/.test(link.getAttribute('href'))) { + event.preventDefault(); navigate(routeUrl('all','',link.getAttribute('href'))); return; + } + const config = event.target.closest('[data-nav-configuration]'); + if(config) { configuration = config.dataset.navConfiguration; history.replaceState(history.state,'',routeUrl('case',caseKey,'#execution-' + configuration)); apply(); } + }); + controls.forEach(control => control.addEventListener(control === query ? 'input' : 'change', () => { history.replaceState(history.state,'',routeUrl(view)); apply(); })); + document.querySelector('[data-report-reset]').addEventListener('click', () => { clearFilters(); history.replaceState(history.state,'',routeUrl(view)); apply(); query.focus(); }); + document.querySelectorAll('[data-report-feature-choice]').forEach(button => button.addEventListener('click', () => { feature.value = button.dataset.reportFeatureChoice; navigate(routeUrl('all','','#results')); document.querySelector('.feature-coverage').open = false; })); + document.addEventListener('keydown', event => { if(event.key === '/' && view !== 'case' && !event.metaKey && !event.ctrlKey && !event.altKey && !event.target.matches('input,select,textarea,[contenteditable]')) { event.preventDefault(); query.focus(); } }); + window.addEventListener('popstate', event => { if(!readRoute())return; restoring = true; apply(); if(typeof event.state?.reportScroll === 'number')window.scrollTo({top:event.state.reportScroll,behavior:'instant'});else if(location.hash)revealTarget(location.hash);else window.scrollTo({top:0,behavior:'instant'});restoring=false; }); + let frame = 0; + window.addEventListener('scroll', () => { if(frame)return;frame=requestAnimationFrame(() => { frame=0;savePosition();const current=sections.find(section=>!section.hidden && section.getBoundingClientRect().top<=130 && section.getBoundingClientRect().bottom>130);document.querySelectorAll('[data-nav-suite-link]').forEach(link=>link.classList.toggle('in-view',view!=='case' && link.dataset.navSuiteLink===current?.dataset.reportSuiteSection)); }); },{passive:true}); + if(readRoute()) { apply(); if(location.hash)revealTarget(location.hash); } +})(); +`; diff --git a/tests/runner-e2e/report-format.ts b/tests/runner-e2e/report-format.ts new file mode 100644 index 00000000000..4b50371ed53 --- /dev/null +++ b/tests/runner-e2e/report-format.ts @@ -0,0 +1,32 @@ +import type { RunnerE2EBillingSummary } from "./types.js"; +import { billingCoverageLabel } from "./billing.js"; + +export function html(value: unknown) { + return String(value ?? "") + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +export function durationLabel(durationMs: number) { + if (durationMs < 1_000) return `${durationMs}ms`; + const seconds = Math.round(durationMs / 1_000); + if (seconds < 60) return `${seconds}s`; + return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; +} + +export function tokenLabel(value: number, llm?: RunnerE2EBillingSummary["llm"]) { + const formatted = new Intl.NumberFormat("en-US").format(value); + return llm ? billingCoverageLabel(formatted, llm.runsWithTokenUsage, llm.runCount) : formatted; +} + +export function llmCostLabel(value: number, llm: RunnerE2EBillingSummary["llm"]) { + return billingCoverageLabel(usdLabel(value), llm.runsWithReportedCost, llm.runCount); +} + +export function usdLabel(value: number | null) { + if (value === null) return "Unknown"; + return `$${value.toFixed(value < 0.01 ? 6 : 4)}`; +} diff --git a/tests/runner-e2e/report-history.ts b/tests/runner-e2e/report-history.ts new file mode 100644 index 00000000000..d83055a4250 --- /dev/null +++ b/tests/runner-e2e/report-history.ts @@ -0,0 +1,311 @@ +import type { RunnerE2EHistoryIndex } from "./types.js"; +import { html, durationLabel, tokenLabel, llmCostLabel, usdLabel } from "./report-format.js"; + +function renderTrendChart(input: { + history: RunnerE2EHistoryIndex; + label: string; + value(campaign: RunnerE2EHistoryIndex["campaigns"][number]): number | null; + format(value: number): string; + include?(campaign: RunnerE2EHistoryIndex["campaigns"][number]): boolean; + fingerprint?(campaign: RunnerE2EHistoryIndex["campaigns"][number]): string; +}) { + const campaigns = input.history.campaigns + .filter(input.include ?? ((campaign) => campaign.complete)) + .slice(0, 20) + .reverse(); + if (campaigns.length === 0) { + return `
${html(input.label)}No complete campaigns
`; + } + const rawValues = campaigns.map(input.value); + const values = rawValues.filter((value): value is number => value !== null); + if (values.length !== rawValues.length) { + const latest = rawValues.at(-1); + return `
${html(input.label)}${latest == null ? "Unknown" : html(input.format(latest))}Unknown measurements are not plotted. Inspect campaign billing and reserved judge budgets.
`; + } + const maximum = Math.max(...values, 1); + const pointRows = values.map((value, index) => { + const x = + campaigns.length === 1 ? 50 : (index / (campaigns.length - 1)) * 100; + const y = 96 - (value / maximum) * 88; + return { + point: `${x.toFixed(2)},${y.toFixed(2)}`, + x, + y, + fingerprint: input.fingerprint?.(campaigns[index]!) ?? "stable", + }; + }); + const segments = pointRows.reduce>( + (groups, point) => { + const current = groups.at(-1); + if (!current || current.at(-1)?.fingerprint !== point.fingerprint) { + groups.push([point]); + } else { + current.push(point); + } + return groups; + }, + [], + ); + const definitionCount = new Set(pointRows.map((point) => point.fingerprint)) + .size; + const latest = values.at(-1) ?? 0; + return `
+ ${html(input.label)} + ${html(input.format(latest))} + + ${segments.map((segment) => ``).join("")} + ${pointRows.map((point) => ``).join("")} + + ${campaigns.length} complete campaign${campaigns.length === 1 ? "" : "s"} · ${definitionCount} definition${definitionCount === 1 ? "" : "s"} +
`; +} + +function suiteSummaryFor( + campaign: RunnerE2EHistoryIndex["campaigns"][number], + suiteId: string, +) { + return campaign.suites.find((suite) => suite.suiteId === suiteId); +} + +export function renderReportHistory(history: RunnerE2EHistoryIndex | undefined) { + if (!history || history.campaigns.length === 0) { + return `

History

Campaign trends

No historical campaigns have been published yet.

`; + } + const suiteIds = [ + ...new Set( + history.campaigns.flatMap((campaign) => + campaign.suites.map((suite) => suite.suiteId), + ), + ), + ]; + const charts = [ + renderTrendChart({ + history, + label: "Observed + estimated cost", + value: (campaign) => campaign.billing.observedAndEstimatedCostUsd, + format: usdLabel, + fingerprint: (campaign) => + campaign.suites + .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) + .sort() + .join("|"), + }), + renderTrendChart({ + history, + label: "Total tokens", + value: (campaign) => campaign.billing.llm.totalTokens, + format: tokenLabel, + fingerprint: (campaign) => + campaign.suites + .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) + .sort() + .join("|"), + }), + renderTrendChart({ + history, + label: "Agent execution time", + value: (campaign) => campaign.billing.agentRunDurationMs, + format: durationLabel, + fingerprint: (campaign) => + campaign.suites + .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) + .sort() + .join("|"), + }), + renderTrendChart({ + history, + label: "Daytona lease time", + value: (campaign) => campaign.billing.leaseDurationMs, + format: durationLabel, + fingerprint: (campaign) => + campaign.suites + .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) + .sort() + .join("|"), + }), + renderTrendChart({ + history, + label: "Pass rate", + value: (campaign) => + campaign.selected > 0 ? (campaign.passed / campaign.selected) * 100 : 0, + format: (value) => `${value.toFixed(1)}%`, + fingerprint: (campaign) => + campaign.suites + .map((suite) => `${suite.suiteId}:${suite.suiteDefinitionHash}`) + .sort() + .join("|"), + }), + ].join(""); + const suiteCharts = suiteIds + .map((suiteId) => { + const include = (campaign: RunnerE2EHistoryIndex["campaigns"][number]) => + suiteSummaryFor(campaign, suiteId)?.complete === true; + const value = ( + campaign: RunnerE2EHistoryIndex["campaigns"][number], + metric: "cost" | "tokens" | "agent" | "lease" | "passRate", + ) => { + const suite = suiteSummaryFor(campaign, suiteId); + if (!suite) return 0; + if (metric === "cost") return suite.billing.observedAndEstimatedCostUsd; + if (metric === "tokens") return suite.billing.llm.totalTokens; + if (metric === "agent") return suite.billing.agentRunDurationMs; + if (metric === "lease") return suite.billing.leaseDurationMs; + return suite.selected > 0 ? (suite.passed / suite.selected) * 100 : 0; + }; + const fingerprint = ( + campaign: RunnerE2EHistoryIndex["campaigns"][number], + ) => suiteSummaryFor(campaign, suiteId)?.suiteDefinitionHash ?? "unknown"; + return `
+

${html(suiteId)}

Complete suite selections only; lines break at definition changes.
+
+ ${renderTrendChart({ history, label: "Suite cost", value: (campaign) => value(campaign, "cost"), format: usdLabel, include, fingerprint })} + ${renderTrendChart({ history, label: "Suite tokens", value: (campaign) => value(campaign, "tokens"), format: tokenLabel, include, fingerprint })} + ${renderTrendChart({ history, label: "Suite agent time", value: (campaign) => value(campaign, "agent"), format: durationLabel, include, fingerprint })} + ${renderTrendChart({ history, label: "Suite lease time", value: (campaign) => value(campaign, "lease"), format: durationLabel, include, fingerprint })} + ${renderTrendChart({ history, label: "Suite pass rate", value: (campaign) => value(campaign, "passRate"), format: (metric) => `${metric.toFixed(1)}%`, include, fingerprint })} +
+
`; + }) + .join(""); + const rows = history.campaigns + .map((campaign) => { + const status = campaign.failed > 0 ? "failed" : (campaign.incomplete ?? 0) > 0 ? "incomplete" : "passed"; + const sha = campaign.source.sha; + const searchable = [ + campaign.campaignId, + sha, + campaign.source.ref, + ...campaign.executions.flatMap((execution) => [ + execution.suiteId, + execution.profileId, + execution.model, + execution.environmentId, + execution.caseId, + execution.status, + ]), + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); + return ` + ${html(campaign.campaignId)}${html(new Date(campaign.generatedAt).toLocaleString("en-US", { timeZone: "UTC" }))} UTC + ${sha ? `${html(sha.slice(0, 10))}` : "Unknown"}${html(campaign.source.ref ?? "unknown ref")} + ${status}${campaign.passed}/${campaign.selected} passed${campaign.incomplete ? ` · ${campaign.incomplete} incomplete` : ""} · ${campaign.complete ? "complete" : "partial"} + ${html(tokenLabel(campaign.billing.llm.inputTokens, campaign.billing.llm))} / ${html(tokenLabel(campaign.billing.llm.outputTokens, campaign.billing.llm))}input / output · ${html(tokenLabel(campaign.billing.llm.cachedInputTokens, campaign.billing.llm))} cached + ${html(llmCostLabel(campaign.billing.reportedLlmCostUsd, campaign.billing.llm))}${html(usdLabel(campaign.billing.estimatedRuntimeCostUsd))} runtime estimate + ${html(durationLabel(campaign.billing.agentRunDurationMs))}${html(durationLabel(campaign.billing.leaseDurationMs))} lease + `; + }) + .join(""); + const latest = history.campaigns.find( + (campaign) => campaign.campaignId === history.latestCampaignId, + ); + const latestGreen = history.campaigns.find( + (campaign) => campaign.campaignId === history.latestGreenCampaignId, + ); + return `
+

History

Campaign trends

Complete campaigns are compared by default. Partial smoke runs remain searchable and are labeled explicitly.

+ +
${charts}
+ ${suiteCharts} +
+ + + + + + +
+
${rows}
CampaignPaperclip SHAResultTokensCostExecution
+ +
`; +} + +export const reportHistoryStyles = ` + .history-pointers { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 12px; margin-bottom: 16px; } + .history-pointers a { padding: 14px 16px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--card); text-decoration: none; } + .history-pointers span, .history-pointers strong { display: block; } + .history-pointers span { color: var(--muted-foreground); font-size: 10px; letter-spacing: .06em; text-transform: uppercase; } + .history-pointers strong { margin-top: 4px; font: 550 12px/1.4 var(--font-mono); overflow-wrap: anywhere; } + .trend-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; } + .trend-card { min-width: 0; padding: 14px 16px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--card); } + .trend-card > span, .trend-card > strong, .trend-card > small { display: block; } + .trend-card > span, .trend-card > small { color: var(--muted-foreground); font-size: 10px; } + .trend-card > span { letter-spacing: .06em; text-transform: uppercase; } + .trend-card > strong { margin-top: 4px; font: 550 17px/1.3 var(--font-mono); } + .trend-card svg { width: 100%; height: 72px; margin: 10px 0 4px; overflow: visible; } + .trend-card polyline { stroke: var(--foreground); stroke-width: 1.6; } + .trend-card circle { fill: var(--background); stroke: var(--foreground); stroke-width: 1; } + .suite-trends { margin-top: 28px; padding-top: 20px; border-top: 1px solid var(--border); } + .suite-trends-heading { display: flex; align-items: baseline; justify-content: space-between; gap: 16px; margin-bottom: 12px; } + .suite-trends-heading h3 { margin: 0; font: 600 14px/1.4 var(--font-mono); } + .suite-trends-heading span { color: var(--muted-foreground); font-size: 10px; } + .history-filters { display: grid; grid-template-columns: minmax(220px, 1fr) repeat(4, minmax(130px, 180px)) auto; align-items: end; gap: 12px; margin: 24px 0 12px; } + .history-filters label { display: grid; gap: 5px; color: var(--muted-foreground); font-size: 10px; letter-spacing: .05em; text-transform: uppercase; } + .history-filters input, .history-filters select { width: 100%; min-height: 38px; padding: 7px 9px; border: 1px solid var(--border-strong); border-radius: calc(var(--radius) * .8); background: var(--background); color: var(--foreground); } + .history-filters .history-checkbox { display: flex; align-items: center; min-height: 38px; padding-bottom: 7px; white-space: nowrap; } + .history-checkbox input { width: 16px; min-height: 16px; } + .history-table-wrap { overflow-x: auto; border-top: 1px solid var(--border); border-bottom: 1px solid var(--border); } + .history-table { width: 100%; min-width: 1020px; border-collapse: collapse; } + .history-table th, .history-table td { padding: 12px; border-bottom: 1px solid var(--border); text-align: left; vertical-align: top; } + .history-table tr:last-child td { border-bottom: 0; } + .history-table th { background: var(--raised); color: var(--muted-foreground); font-size: 10px; letter-spacing: .05em; text-transform: uppercase; } + .history-table td small { display: block; margin-top: 3px; color: var(--muted-foreground); font-size: 10px; } + .history-table .status { margin-bottom: 2px; } + .history-passed { border-color: var(--pass-border); background: var(--pass-bg); color: var(--pass-text); } + .history-incomplete { border-color: var(--missing-border); background: var(--missing-bg); color: var(--missing-text); } + .history-failed { border-color: var(--fail-border); background: var(--fail-bg); color: var(--fail-text); } + .history-empty { padding: 24px 0; color: var(--muted-foreground); text-align: center; } + .history-section .section-heading { display:block; margin:24px 0; } + .history-section .section-heading h2 { font-size:24px; margin:0; } + .history-section .section-heading p { color:var(--muted-foreground); font-size:12px; } + .history-section .status { border:1px solid; } + [data-index-trends] { margin-top:32px; } + [data-index-trends] > summary { cursor:pointer; color:var(--muted-foreground); font-size:12px; } + @media(max-width:1100px) { .trend-grid { grid-template-columns:repeat(2,minmax(0,1fr)); }.history-filters { grid-template-columns:repeat(2,minmax(0,1fr)); } } + @media(max-width:760px) { .trend-grid,.history-pointers,.history-filters { grid-template-columns:1fr; }.history-table-wrap .history-table { display:table; min-width:1020px; }.history-table tbody { display:table-row-group; }.history-table tr { display:table-row; }.history-table th,.history-table td { display:table-cell; }.history-table td::before { display:none; } } +`; + +export const reportHistoryScript = ` + (() => { + const rows = Array.from(document.querySelectorAll("[data-history-campaign]")); + if (rows.length === 0) return; + const query = document.querySelector("[data-history-query]"); + const suite = document.querySelector("[data-history-suite]"); + const status = document.querySelector("[data-history-status]"); + const from = document.querySelector("[data-history-from]"); + const through = document.querySelector("[data-history-through]"); + const partial = document.querySelector("[data-history-partial]"); + const empty = document.querySelector("[data-history-empty]"); + const suiteTrends = [...document.querySelectorAll("[data-history-suite-trends]")]; + const apply = () => { + const search = query.value.trim().toLowerCase(); + let visible = 0; + rows.forEach((row) => { + const matches = + (!search || row.dataset.historySearch.includes(search)) && + (!suite.value || row.dataset.historySuites.split(" ").includes(suite.value)) && + (!status.value || row.dataset.historyStatus === status.value) && + (!from.value || row.dataset.historyDate >= from.value) && + (!through.value || row.dataset.historyDate <= through.value) && + (partial.checked || row.dataset.historyComplete === "true"); + row.hidden = !matches; + if (matches) visible += 1; + }); + suiteTrends.forEach((section) => { + section.hidden = Boolean(suite.value) && section.dataset.historySuiteTrends !== suite.value; + }); + empty.hidden = visible !== 0; + }; + query.addEventListener("input", apply); + suite.addEventListener("change", apply); + status.addEventListener("change", apply); + from.addEventListener("change", apply); + through.addEventListener("change", apply); + partial.addEventListener("change", apply); + apply(); + })(); +`; diff --git a/tests/runner-e2e/report-navigation.test.ts b/tests/runner-e2e/report-navigation.test.ts new file mode 100644 index 00000000000..44ecb6cd657 --- /dev/null +++ b/tests/runner-e2e/report-navigation.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { renderRunnerE2EDashboard } from "./dashboard.js"; +import { buildRunnerCampaign, campaignHistoryRecord, emptyRunnerHistory, mergeRunnerHistory } from "./history.js"; +import { renderRunnerHistoryIndex } from "./history-index.js"; +import { renderReportShell, reportHistoryHref } from "./report-navigation.js"; + +describe("evaluation report navigation", () => { + it("never gives global pages a campaign suite tree or campaign selector", () => { + const campaign = buildRunnerCampaign({campaignId:"global-fixture",generatedAt:"2026-10-20T12:00:00Z",expected:[runnerMatrix[0]!.id],results:[]}); + const history = mergeRunnerHistory(null,campaignHistoryRecord(campaign,"https://reports.example.test/runner-e2e")); + const page = renderRunnerHistoryIndex(history); + const sidebar = page.slice(page.indexOf('