diff --git a/maps/feature-map/README.md b/maps/feature-map/README.md index 545175a7f5d..72f06363c27 100644 --- a/maps/feature-map/README.md +++ b/maps/feature-map/README.md @@ -178,6 +178,11 @@ describe documentation scope, not runtime health. The map documents user journeys. It does not execute them automatically; generated inventories record documentation scope rather than live test health. +Product E2E reports reuse these capability IDs through the +[journey associations](../../tests/runner-e2e/journey-features.ts). Their coverage +view shows which scenarios in a campaign are linked to each capability and the +associated test results. An association or passing scenario does not certify +the whole capability; unselected and unmapped journeys remain distinct. When maintaining a recipe, compare its entry points and expected results with the current source. Check that linked tests still exercise the stated behavior diff --git a/scripts/feature-map.d.mts b/scripts/feature-map.d.mts new file mode 100644 index 00000000000..bdced13d85f --- /dev/null +++ b/scripts/feature-map.d.mts @@ -0,0 +1,16 @@ +export interface FeatureCapability { + id: string; + label: string; + description: string; + verification?: { scope: string; storybook: string; production: string; sources: string }; +} +export interface FeatureMapIndex { + sections: Array<{ + title: string; + features: Array<{ id: string; title: string; description: string; capabilities: FeatureCapability[] }>; + }>; +} +export function featureIndex(root: string): FeatureMapIndex; +export function sidebarLabel(title: string): string; +export function mapIndexes(index: FeatureMapIndex): Map; +export function pageInventory(root: string, index: FeatureMapIndex): { pages: Array<{ file: string; features: string[] }> }; diff --git a/tests/e2e/runner-e2e-dashboard.spec.ts b/tests/e2e/runner-e2e-dashboard.spec.ts index 8a82e833c4b..f29ca3b04b7 100644 --- a/tests/e2e/runner-e2e-dashboard.spec.ts +++ b/tests/e2e/runner-e2e-dashboard.spec.ts @@ -1,10 +1,11 @@ import { expect, test } from "@playwright/test"; -import { runnerExecutionById } from "../runner-e2e/catalog.js"; +import { runnerExecutionById, runnerMatrix } from "../runner-e2e/catalog.js"; import { renderRunnerE2EDashboard, type RunnerDashboardEntry, } from "../runner-e2e/dashboard.js"; +import { buildRunnerCampaign } from "../runner-e2e/history.js"; import type { MatrixExecution, RunnerE2EResult } from "../runner-e2e/types.js"; const executionIds = [ @@ -77,7 +78,7 @@ test("filters matrix results and pages through the filtered screenshot gallery", title: "Runner E2E interaction fixture", generatedAt: "2026-09-05T12:01:00.000Z", expected: executionIds, - catalog, + catalog: runnerMatrix, entries, }), ); @@ -86,19 +87,36 @@ test("filters matrix results and pages through the filtered screenshot gallery", .locator("[data-report-query]") .locator("..") .locator(".."); - const tabs = page.locator(".suite-nav"); + await expect(page.locator(".suite-nav")).toHaveCount(0); + await expect(page.locator(".suite-section")).toHaveCount(2); + await expect(page.locator('select[data-report-suite] option')).toHaveCount(3); + await expect(page.locator('select[data-report-profile] option')).toHaveCount(4); await expect(page.locator("[data-report-result]")).toHaveText( "4 of 4 tests shown", ); await expect(page.locator("[data-gallery-item]")).toHaveCount(4); + const passedCard = page.locator('[data-execution-id="core-compatibility.legacy-codex.local.message-marker"]'); + const failedCard = page.locator('[data-execution-id="core-compatibility.legacy-codex.daytona.message-marker"]'); + await expect(passedCard.locator(".status")).toHaveText("passed"); + await expect(passedCard.locator(".case-outcome")).toBeHidden(); + await expect(passedCard.locator(".billing-strip")).toBeHidden(); + await expect(passedCard.locator(".execution-id")).toBeHidden(); + await expect(passedCard.locator(".case-diagnostics > summary")).toContainText("1/1 checks · 12s"); + await expect(failedCard.locator(":scope > .case-outcome")).toBeVisible(); expect( await filters.evaluate((element) => getComputedStyle(element).position), ).toBe("static"); - expect( - await tabs.evaluate((element) => - element.nextElementSibling?.classList.contains("report-filters"), - ), - ).toBe(true); + await page.locator('.feature-coverage > summary').click(); + await page.locator('.feature-coverage').getByRole('button', { name: 'Questions and approvals', exact: true }).click(); + await expect(page.locator('select[data-report-feature]')).toHaveValue('questions-and-approvals'); + await expect(page.locator('.feature-coverage')).not.toHaveAttribute('open'); + await expect(page.locator('[data-report-result]')).toHaveText('1 of 4 tests shown'); + await expect(page.locator('.suite-section:visible')).toHaveCount(1); + await expect(page.locator('[data-gallery-open]')).toHaveText('View gallery · 1'); + await page.setViewportSize({ width: 900, height: 900 }); + await expect(page.locator('[data-report-profile-row]:visible')).toHaveCount(1); + await page.setViewportSize({ width: 1280, height: 720 }); + await page.locator('[data-report-reset]').click(); const firstTable = page.locator(".matrix").first(); const widthsBefore = await firstTable @@ -113,6 +131,7 @@ test("filters matrix results and pages through the filtered screenshot gallery", cells.map((cell) => cell.getBoundingClientRect().width), ); expect(widthsAfter).toEqual(widthsBefore); + await expect(passedCard.locator(".billing-strip")).toBeVisible(); await page.locator("[data-report-query]").fill("Legacy Claude"); await expect(page.locator("[data-report-result]")).toHaveText( @@ -146,6 +165,7 @@ test("filters matrix results and pages through the filtered screenshot gallery", await expect(dialog.locator("[data-gallery-position]")).toHaveText( "Image 1 of 2", ); + await expect(dialog.locator("[data-gallery-original]")).toHaveAttribute("href", /legacy-codex\.local\.message-marker\/attempt-1\/final-state\.png$/); await page.keyboard.press("ArrowRight"); await expect(dialog.locator("[data-gallery-environment]")).toHaveText( @@ -155,6 +175,7 @@ test("filters matrix results and pages through the filtered screenshot gallery", await expect(dialog.locator("[data-gallery-position]")).toHaveText( "Image 2 of 2", ); + await expect(dialog.locator("[data-gallery-original]")).toHaveAttribute("href", /legacy-codex\.daytona\.message-marker\/attempt-1\/final-state\.png$/); await page.keyboard.press("Escape"); await expect(dialog).not.toBeVisible(); @@ -162,3 +183,158 @@ test("filters matrix results and pages through the filtered screenshot gallery", await page.keyboard.press("/"); await expect(page.locator("[data-report-query]")).toBeFocused(); }); + +function navigationReport(ids: readonly string[], suppliedEntries?: readonly RunnerDashboardEntry[]) { + const catalog = ids.map(runnerExecutionById); + const entries = suppliedEntries ?? catalog.map((execution) => resultFor(execution, "passed")); + const generatedAt = "2026-09-05T12:01:00.000Z"; + return renderRunnerE2EDashboard({ + title: "Runner E2E navigation fixture", + generatedAt, + expected: ids, + catalog: runnerMatrix, + entries, + campaign: buildRunnerCampaign({ + campaignId: "navigation-fixture", + generatedAt, + expected: ids, + catalog, + results: entries.map((entry) => entry.result), + }), + }); +} + +test("suite links reveal filtered sections and scroll on repeated clicks", async ({ page }) => { + const report = navigationReport(executionIds); + const reportUrl = "http://runner-e2e.test/campaign/"; + await page.route(reportUrl, (route) => route.fulfill({ contentType: "text/html", body: report })); + await page.goto(reportUrl); + + const core = page.locator("#suite-core-compatibility"); + const coreHeading = core.getByRole("heading", { name: "Core Runner Compatibility", exact: true }); + const coreLink = page.getByRole("link", { name: "Core Runner Compatibility", exact: true }); + const backToTop = page.getByRole("link", { name: "Back to top", exact: true }); + await expect(backToTop).toBeHidden(); + const feature = page.locator("select[data-report-feature]"); + for (let click = 0; click < 2; click += 1) { + await feature.selectOption("questions-and-approvals"); + await expect(core).toBeHidden(); + await coreLink.click(); + await expect(core).toBeVisible(); + await expect(coreHeading).toBeInViewport(); + await expect(feature).toHaveValue(""); + await expect(page).toHaveURL(`${reportUrl}#suite-core-compatibility`); + await expect(page.locator("[data-report-result]")).toHaveText("4 of 4 tests shown"); + } + + await page.locator("select[data-report-profile]").selectOption("legacy-codex"); + await coreLink.click(); + await expect(coreHeading).toBeInViewport(); + await expect(page.locator("select[data-report-profile]")).toHaveValue("legacy-codex"); + await expect(backToTop).toBeVisible(); + expect(await backToTop.evaluate((element) => getComputedStyle(element).position)).toBe("fixed"); + await backToTop.focus(); + await page.keyboard.press("Enter"); + await expect.poll(() => page.evaluate(() => window.scrollY)).toBe(0); + await expect(backToTop).toBeHidden(); + await expect(page.locator(".report-header h1")).toBeFocused(); + await expect(page.locator("select[data-report-profile]")).toHaveValue("legacy-codex"); + await coreLink.click(); + + await page.getByRole("link", { name: "Local Session Integrity", exact: true }).click(); + await expect(page.locator("#suite-local-session-integrity")).toBeVisible(); + await feature.selectOption("questions-and-approvals"); + await page.goBack(); + await expect(coreHeading).toBeInViewport(); + await expect(feature).toHaveValue(""); +}); + +test("folds the suite index and links each case to a matching configuration", async ({ page }) => { + const ids = [...executionIds, "core-compatibility.legacy-codex.local.ask-question"]; + const reportUrl = "http://runner-e2e.test/campaign/"; + await page.route(reportUrl, (route) => route.fulfill({ contentType: "text/html", body: navigationReport(ids) })); + await page.goto(reportUrl); + const coreIndex = page.locator(".campaign-suite-details").filter({ + has: page.getByRole("link", { name: "Core Runner Compatibility", exact: true }), + }); + await expect(coreIndex).not.toHaveAttribute("open"); + await coreIndex.getByRole("link", { name: "Core Runner Compatibility", exact: true }).click(); + await expect(coreIndex).not.toHaveAttribute("open"); + await coreIndex.locator("summary").focus(); + await page.keyboard.press("Enter"); + await expect(coreIndex).toHaveAttribute("open"); + await expect(coreIndex.locator(".suite-case-list a")).toHaveCount(2); + const sectionToc = page.locator("#suite-core-compatibility .suite-case-toc"); + await expect(sectionToc.locator(".suite-case-list a")).toHaveCount(2); + + const basicLink = coreIndex.getByRole("link", { name: "Basic response", exact: true }); + const profile = page.locator("select[data-report-profile]"); + const feature = page.locator("select[data-report-feature]"); + await profile.selectOption("legacy-claude"); + const claudeResult = page.locator('[data-execution-id="core-compatibility.legacy-claude.local.message-marker"]'); + await basicLink.click(); + await expect(claudeResult.locator(".case-heading")).toBeInViewport(); + await expect(profile).toHaveValue("legacy-claude"); + await expect(page).toHaveURL(`${reportUrl}#execution-core-compatibility.legacy-claude.local.message-marker`); + + for (let click = 0; click < 2; click += 1) { + await feature.selectOption("questions-and-approvals"); + await basicLink.click(); + await expect(feature).toHaveValue(""); + await expect(profile).toHaveValue(""); + await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.local.message-marker"] .case-heading')).toBeInViewport(); + } + await sectionToc.getByRole("link", { name: "Ask mode question", exact: true }).click(); + await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.local.ask-question"] .case-heading')).toBeInViewport(); + await sectionToc.getByRole("link", { name: "Back to index", exact: false }).click(); + await expect(coreIndex.locator("summary")).toBeInViewport(); + + await page.setViewportSize({ width: 390, height: 844 }); + await expect(basicLink).toBeVisible(); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await basicLink.click(); + const backToTop = page.getByRole("link", { name: "Back to top", exact: true }); + await expect(backToTop).toBeVisible(); + expect(await backToTop.evaluate((element) => { + const bounds = element.getBoundingClientRect(); + return bounds.left >= 0 && bounds.right <= innerWidth && bounds.bottom <= innerHeight; + })).toBe(true); + await backToTop.click(); + await expect.poll(() => page.evaluate(() => window.scrollY)).toBe(0); +}); + +test("case status dots summarize matching results and open failures", async ({ page }) => { + const ids = [...executionIds, "core-compatibility.legacy-codex.local.ask-question"]; + const entries = executionIds.map((id, index) => resultFor(runnerExecutionById(id), index === 1 ? "failed" : "passed")); + const reportUrl = "http://runner-e2e.test/campaign/"; + await page.route(reportUrl, (route) => route.fulfill({ contentType: "text/html", body: navigationReport(ids, entries) })); + await page.goto(reportUrl); + const toc = page.locator("#suite-core-compatibility .suite-case-toc"); + const basic = toc.getByRole("link", { name: "Basic response", exact: true }); + const missing = toc.getByRole("link", { name: "Ask mode question", exact: true }); + const topLink = page.locator('.campaign-case-index [data-report-case-link="core-compatibility.message-marker"]'); + await expect(basic).toHaveAttribute("data-case-status", "failed"); + await expect(topLink).toHaveAttribute("data-case-status", "failed"); + await expect(basic).toHaveAccessibleDescription("Failed · 2/3 passed · 1 failed — All configurations"); + await expect(missing).toHaveAttribute("data-case-status", "missing"); + await expect(missing).toHaveAttribute("title", "Missing result · 0/1 passed · 1 missing — All configurations"); + const red = await basic.locator(".case-status-dot").evaluate((element) => getComputedStyle(element).backgroundColor); + + const profile = page.locator("select[data-report-profile]"); + await profile.selectOption("legacy-claude"); + await expect(basic).toHaveAttribute("data-case-status", "passed"); + await expect(topLink).toHaveAttribute("data-case-status", "passed"); + await expect(basic).toHaveAccessibleDescription("Passed · 1/1 passed — Matching configurations"); + const green = await basic.locator(".case-status-dot").evaluate((element) => getComputedStyle(element).backgroundColor); + expect(green).not.toBe(red); + await expect(missing).toHaveAttribute("title", /none match current filters/); + + await profile.selectOption("legacy-codex"); + await expect(basic).toHaveAttribute("data-case-status", "failed"); + await page.locator("select[data-report-environment]").selectOption("local"); + await expect(basic).toHaveAttribute("data-case-status", "passed"); + await page.locator("[data-report-reset]").click(); + await basic.click(); + await expect(page).toHaveURL(`${reportUrl}#execution-core-compatibility.legacy-codex.daytona.message-marker`); + await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.daytona.message-marker"] .case-heading')).toBeInViewport(); +}); diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 24c1cf4b7d2..f9ff0e2223b 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -253,6 +253,28 @@ pnpm test:e2e:runner -- --list --suite completion-updates pnpm test:e2e:runner -- --suite completion-updates ``` +## Local report design preview + +Iterate on the public history and individual campaign reports with retained real +data, without starting Paperclip, loading credentials, or calling providers: + +```sh +node --watch --import ./cli/node_modules/tsx/dist/loader.mjs tests/runner-e2e/preview.ts +``` + +Open `http://127.0.0.1:4317/runner-e2e/`. The preview uses the production +`history-index.ts` and `dashboard.ts` renderers. Changes restart the preview +and reload the browser. Every history link opens a locally rendered report; +normalized results, public screenshots, and other manifest-listed public evidence +are downloaded on first access and cached under the ignored +`tests/runner-e2e/results/design-preview/` directory. Original grades, billing, +timestamps, and source provenance are retained. No results are republished. + +Use `--port 4320` for another port, `--refresh` to refresh the history snapshot, +or `--offline` to require cached data. Offline mode supports reports and assets +already visited; it reports uncached artifacts explicitly. The preview contains +only published evidence, not private Playwright traces or videos. + ## Credentials Copy `.env.runner-e2e.example` to `.env.runner-e2e.local` and fill only the @@ -757,7 +779,41 @@ captures remain failed evidence; qualifying a fix requires a new live attempt. Every completed local campaign also writes `tests/runner-e2e/results//dashboard.html`. The self-contained page shows the complete profile/environment grid with screenshot thumbnails. -Expanding a case shows its matchers, pass/fail details, provider/model/runtime, +The history homepage and campaign report render their campaign summaries as +selectable HTML text, with suite links, pass counts, status, and timing. +Campaign reports render only selected executions, including selected cases with +missing evidence or startup failures. Search and feature/suite/profile filters +replace the full-catalog sticky index. +Each suite in the campaign report summary folds open to a list of its selected cases. +The same case links appear at the start of the suite, with a return to the top +index. Cases appear once across agent/environment configurations; links prefer +a result matching the active filters and clear conflicting filters when needed +to reveal their destination. +Case links show a green status dot when all matching configurations passed, +red when any failed, and amber for incomplete or missing results. Their tooltip +summarizes the configurations behind the status; red links prefer a failing +result. The status follows the active filters, falling back to the whole case +when none of its configurations match. + +`journey-features.ts` associates exact suite/case identities with the stable +capability IDs from `maps/feature-map/`. The feature selector uses the map's +existing sections and titles. The expandable Feature coverage view separates +capability associations, linked test outcomes, gaps, and unmapped journeys. +These display-only associations do not change prompts, graders, definitions, +or retained results. Historical reports use the current associations, not a +claim that those annotations were recorded by the original run. A linked, +passing scenario does not establish complete capability coverage. Extend the +associations only after inspecting the journey's assertions; unknown cases stay +unmapped rather than inheriting coverage from a similarly named suite. + +Passing cards emphasize screenshots and show checks and duration beside a +single Test details disclosure. That disclosure holds billing, evidence links, +execution identity, and diagnostics; failed and incomplete results keep their +explanations visible. Screenshots open the gallery, which also links to the +original image. A floating Back to top control returns to the report heading +without clearing filters. + +Expanding Test details shows matchers, pass/fail details, provider/model/runtime, timings, token and cost accounting, and evidence links. The campaign header aggregates input, output, and cached tokens, provider-reported LLM spend, Daytona list-price runtime estimates, and pricing coverage. Missing provider diff --git a/tests/runner-e2e/campaign-summary.ts b/tests/runner-e2e/campaign-summary.ts new file mode 100644 index 00000000000..8e7012f4943 --- /dev/null +++ b/tests/runner-e2e/campaign-summary.ts @@ -0,0 +1,85 @@ +import type { ReportExecution } from "./report-catalog.js"; +import type { RunnerE2ESuiteSummary } from "./types.js"; + +function html(value: unknown) { + return String(value ?? "").replace(/[&<>"']/g, (character) => + ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[character]!, + ); +} + +function uniqueCases(catalog: readonly ReportExecution[]) { + const cases = new Map(); + for (const execution of catalog) { + const key = `${execution.suite.id}.${execution.task.id}`; + if (!cases.has(key)) cases.set(key, execution); + } + return [...cases.values()]; +} + +export function renderSuiteCaseIndex(catalog: readonly ReportExecution[]) { + return ``; +} + +export function renderCampaignSuiteSummary( + suites: readonly RunnerE2ESuiteSummary[], + catalog: readonly ReportExecution[], + reportHref = "", + showCases = false, +) { + const labels = new Map(catalog.map((execution) => [execution.suite.id, execution.suite.label])); + const selected = suites.filter((suite) => suite.selected > 0); + if (selected.length === 0) return ""; + return `
    ${selected.map((suite) => { + const passed = suite.passed === suite.selected && suite.cleanupPassed; + const label = labels.get(suite.suiteId) ?? suite.suiteId; + const suiteCatalog = catalog.filter((execution) => execution.suite.id === suite.suiteId); + const caseCount = uniqueCases(suiteCatalog).length; + const fold = showCases && caseCount > 0; + const heading = `${html(label)}${fold ? `${caseCount} ${caseCount === 1 ? "case" : "cases"}` : ""} + ${suite.passed}/${suite.selected} + ${passed ? "Passed" : "Needs attention"}${suite.incomplete ? ` · ${suite.incomplete} incomplete` : ""}`; + return `
  • ${fold ? `
    + ${heading} + +
    ` : `
    ${heading}
    `} +
    • `; + }).join("")}
`; +} + +export const campaignSuiteSummaryStyles = ` + .campaign-suites { display: grid; margin: 0 0 32px; padding: 0; list-style: none; } + .campaign-suites > li { border-bottom: 1px solid var(--border); } + .campaign-suite-row { display: grid; grid-template-columns: minmax(0, 1fr) auto 150px; align-items: baseline; gap: 24px; padding: 12px 0; } + .campaign-suite-details > summary { grid-template-columns: 16px minmax(0, 1fr) auto 150px; gap: 12px; cursor: pointer; list-style: none; } + .campaign-suite-details > summary::-webkit-details-marker { display: none; } + .campaign-suite-details > summary::before { content: "›"; align-self: center; color: var(--muted-foreground, var(--muted)); font-size: 20px; line-height: 1; text-align: center; } + .campaign-suite-details[open] > summary::before { transform: rotate(90deg); } + .campaign-suite-title { display: flex; flex-wrap: wrap; align-items: baseline; gap: 4px 12px; min-width: 0; } + .campaign-suite-title small { color: var(--muted-foreground, var(--muted)); font-size: 11px; } + .campaign-case-index { padding: 0 0 16px 28px; } + .campaign-suites a { width: fit-content; font-weight: 500; text-decoration: none; } + .campaign-suites a:hover { text-decoration: underline; } + .campaign-suites strong { font-family: var(--font-mono, var(--mono)); font-variant-numeric: tabular-nums; font-weight: 600; } + .campaign-suite-state { font-size: 12px; text-align: right; } + .campaign-suite-passed { color: var(--pass-text, var(--pass)); } + .campaign-suite-attention { color: var(--fail-text, var(--fail)); } + .suite-case-list { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 6px 24px; margin: 0; padding: 0; list-style: none; } + .suite-case-list a { display: inline-flex; align-items: baseline; gap: 8px; padding: 4px 0; color: var(--muted-foreground, var(--muted)); font-size: 12px; font-weight: 400; text-decoration: none; text-underline-offset: 3px; overflow-wrap: anywhere; } + .suite-case-list a:hover { color: var(--foreground); text-decoration: underline; } + .case-status-dot { display: inline-block; flex: 0 0 7px; width: 7px; height: 7px; border-radius: 50%; background: var(--muted-foreground, var(--muted)); } + [data-case-status="passed"] .case-status-dot { background: var(--pass-text, var(--pass)); } + [data-case-status="failed"] .case-status-dot { background: var(--fail-text, var(--fail)); } + [data-case-status="incomplete"] .case-status-dot, [data-case-status="missing"] .case-status-dot { background: var(--missing-text); } + @media (max-width: 980px) { + .suite-case-list { grid-template-columns: repeat(2, minmax(0, 1fr)); } + } + @media (max-width: 520px) { + .campaign-suite-row { grid-template-columns: minmax(0, 1fr) auto; gap: 4px 16px; } + .campaign-suite-state { grid-column: 1 / -1; text-align: left; } + .campaign-suite-details > summary { grid-template-columns: 16px minmax(0, 1fr) auto; gap: 4px 12px; } + .campaign-suite-details .campaign-suite-state { grid-column: 2 / -1; } + .suite-case-list { grid-template-columns: 1fr; gap: 2px; } + } +`; diff --git a/tests/runner-e2e/dashboard-regenerate.ts b/tests/runner-e2e/dashboard-regenerate.ts index 4c59220fad8..96976b8b9b7 100644 --- a/tests/runner-e2e/dashboard-regenerate.ts +++ b/tests/runner-e2e/dashboard-regenerate.ts @@ -57,7 +57,6 @@ export async function regenerateRunnerDashboard(input: { historyFile?: string | null; outputDirectory?: string; evidenceHrefPrefix?: string; - publicSummaryImageHref?: string; }) { const bundle = path.resolve(input.bundle); const outputDirectory = path.resolve(input.outputDirectory ?? bundle); @@ -158,7 +157,6 @@ export async function regenerateRunnerDashboard(input: { entries, campaign, history, - publicSummaryImageHref: input.publicSummaryImageHref, }); const upgraded = { ...campaign, diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 039900d6a82..28e8c0e0003 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -1,4 +1,7 @@ import { isIncompleteFirstTaskResult, renderCaseOutcome } from "./case-outcome.js"; +import { campaignSuiteSummaryStyles, renderCampaignSuiteSummary, renderSuiteCaseIndex } from "./campaign-summary.js"; +import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js"; +import { journeyFeatureStyles, renderJourneyFeatureCoverage, renderJourneyFeatureOptions } from "./feature-coverage.js"; import { STARTUP_REUSE_LAYERS, summarizeStartupReuse } from "../../packages/adapter-utils/src/startup-reuse.js"; import { renderFirstTaskDetails } from "./first-task-report.js"; import { @@ -34,7 +37,6 @@ export interface RunnerDashboardInput { entries: readonly RunnerDashboardEntry[]; campaign?: RunnerE2ECampaign; history?: RunnerE2EHistoryIndex; - publicSummaryImageHref?: string; } interface ResolvedScreenshot { @@ -89,17 +91,6 @@ function safeEvidenceHref(base: string | undefined, relative: string) { return `${cleanBase}/${cleanRelative}`; } -function safePublicAssetHref(relative: string | undefined) { - if (!relative || /^(?:[a-z]+:|\/\/|\/)/i.test(relative)) return null; - const segments = relative.split("/"); - if ( - segments.some((segment) => !segment || segment === "." || segment === "..") - ) { - return null; - } - return segments.map(encodeURIComponent).join("/"); -} - function compactJson(value: unknown) { const serialized = JSON.stringify(value); return serialized && serialized.length > 1_200 @@ -178,6 +169,10 @@ function renderCase( : item, ); const billing = entry ? summarizeExecutionBilling(entry.result) : null; + const capabilities = journeyCapabilities(execution); + const features = [...new Map(capabilities.map((capability) => [capability.feature.id, capability.feature])).values()]; + const outcome = entry ? renderCaseOutcome(entry.result, entry.valid, entry.errors) : ""; + const passed = state === "passed" && entry?.result.status === "passed"; const firstTaskChecks = entry?.result.firstTask?.checks; const notReached = firstTaskChecks?.filter((c) => c.notReached).length ?? 0; const matcherResults = firstTaskChecks?.length @@ -200,6 +195,7 @@ function renderCase( execution.environment.label, execution.environment.provider, execution.suite.label, + ...capabilities.flatMap((capability) => [capability.id, capability.label, capability.feature.title]), label, ] .join(" ") @@ -222,9 +218,8 @@ function renderCase( return href ? [`${html(label)}`] : []; }); const links = - screenshots.length > 0 || playwright || structuredLinks.length > 0 + playwright || structuredLinks.length > 0 ? `` @@ -302,7 +297,9 @@ function renderCase( class="case case-${state}" data-execution-id="${html(execution.id)}" data-report-case + data-report-case-key="${html(`${execution.suite.id}.${execution.task.id}`)}" data-report-suite="${html(execution.suite.id)}" + data-report-features="${html(journeyFeatureIds(execution).join(" "))}" data-report-profile="${html(execution.profile.id)}" data-report-environment="${html(execution.environment.id)}" data-report-status="${html(state)}" @@ -312,14 +309,20 @@ function renderCase( ${html(execution.task.label)} ${html(label)} - ${entry ? renderCaseOutcome(entry.result, entry.valid, entry.errors) : ""} + ${features.length ? `
${features.map((feature) => + ``, + ).join("")}
` : ""} + ${passed ? "" : outcome} + ${!entry ? `

No result artifact was uploaded.

` : ""} ${gallery} +
+ Test details${matcherResults.length ? `${passedMatchers}/${matcherResults.length} checks${notReached ? ` · ${notReached} not reached` : ""}` : "No checks recorded"}${entry ? ` · ${html(durationLabel(entry.result.durationMs))}` : ""} + ${passed ? outcome : ""} ${billingStrip} ${links} ${renderFirstTaskDetails(entry?.result)} ${html(execution.id)} -
- Matchers and test context + ${capabilities.length ? `
    ${capabilities.map((capability) => `
  • ${html(capability.feature.title)} · ${html(capability.label)}
    • `).join("")}
` : ""} ${ entry ? `
@@ -633,6 +636,10 @@ function renderSuiteMatrix(input: { return `

Test suite

${html(suite.label)}

${html(suite.description)}

${summaryHtml} +
Configuration matrix${profiles.length} profiles · ${environments.length} environments · ${selected} selected
${environments.map(() => '').join("")}${environmentHeaders}${rows}
Agent profile
`; @@ -640,12 +647,12 @@ function renderSuiteMatrix(input: { export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { + const expected = new Set(input.expected); const catalog = discoverReportCatalog({ catalog: input.catalog, expected: input.expected, results: input.entries.map((entry) => entry.result), - }); - const expected = new Set(input.expected); + }).filter((execution) => expected.has(execution.id)); const entryById = new Map( input.entries.map((entry) => [entry.result.executionId, entry]), ); @@ -688,9 +695,6 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { ) .join(""); const historySection = renderHistory(input.history); - const publicSummaryImageHref = safePublicAssetHref( - input.publicSummaryImageHref, - ); const filterProfiles = [ ...new Map( catalog.map((execution) => [execution.profile.id, execution.profile]), @@ -748,7 +752,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { --chat-human-foreground: #ffffff; --chat-bubble-radius: 1rem; --chat-body-size: 0.875rem; - --navigation-sticky-offset: 58px; + --navigation-sticky-offset: 0px; --font-sans: "Paperclip Inter", Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; --font-mono: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace; } @@ -780,6 +784,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { } } * { box-sizing: border-box; } + [hidden] { display: none !important; } html { scroll-behavior: smooth; } body { margin: 0; background: var(--background); color: var(--foreground); font: 14px/1.5 var(--font-sans); font-feature-settings: "ss01", "cv11"; } button, summary, a { -webkit-tap-highlight-color: transparent; } @@ -805,7 +810,9 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .gallery-launch { min-width: 138px; padding: 10px 15px; } .gallery-launch:hover, .gallery-control:hover, .gallery-close:hover { border-color: var(--foreground); background: var(--primary); color: var(--primary-foreground); } .gallery-launch:focus-visible, .gallery-control:focus-visible, .gallery-close:focus-visible, .screenshot-trigger:focus-visible, .report-filters input:focus-visible, .report-filters select:focus-visible, .filter-reset:focus-visible, summary:focus-visible, a:focus-visible { outline: 2px solid #2563eb; outline-offset: 3px; } - .report-filters { display: grid; grid-template-columns: minmax(240px, 1.6fr) repeat(4, minmax(130px, .7fr)) auto; align-items: end; gap: 10px; margin: 0 0 40px; padding: 12px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--card); } + .report-filters { display: grid; grid-template-columns: repeat(6, minmax(0, 1fr)); align-items: end; gap: 10px; margin: 0 0 40px; padding: 12px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--card); } + .report-search { grid-column: span 5; } + .report-feature { grid-column: span 2; } .report-filters label { display: grid; gap: 5px; min-width: 0; } .report-filters label > span { color: var(--muted-foreground); font-size: 10px; font-weight: 600; letter-spacing: .05em; text-transform: uppercase; } .report-filters input, .report-filters select, .filter-reset { width: 100%; min-height: 38px; padding: 7px 9px; border: 1px solid var(--border-strong); border-radius: calc(var(--radius) * .8); background: var(--background); color: var(--foreground); } @@ -847,9 +854,10 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .agent-10 { background: linear-gradient(to bottom, #f2d95f, #4fbcba); } .generation { display: inline-block; margin-left: 7px; padding: 2px 7px; border: 1px solid var(--border); border-radius: 999px; color: var(--muted-foreground); font: 500 10px/1.4 var(--font-mono); letter-spacing: .06em; text-transform: uppercase; } .case-stack { display: grid; gap: 12px; } - .case { min-width: 0; max-width: 100%; padding: 16px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--card); overflow: hidden; } + .case { min-width: 0; max-width: 100%; padding: 16px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--card); overflow: hidden; scroll-margin-top: 80px; } .case-heading { display: flex; align-items: start; justify-content: space-between; gap: 12px; } .case-heading strong { font-size: 14px; font-weight: 650; } + .case-empty { margin: 12px 0; color: var(--muted-foreground); font-size: 12px; } .status, .matcher-state { display: inline-block; flex: none; padding: 3px 8px; border: 1px solid; border-radius: 999px; font: 600 10px/1.4 var(--font-sans); letter-spacing: .05em; text-transform: uppercase; } .case-passed .status, .matcher-passed .matcher-state { border-color: var(--pass-border); background: var(--pass-bg); color: var(--pass-text); } .case-failed .status, .matcher-failed .matcher-state { border-color: var(--fail-border); background: var(--fail-bg); color: var(--fail-text); } @@ -899,9 +907,9 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .billing-metric strong { display: block; overflow-wrap: anywhere; font: 550 17px/1.25 var(--font-mono); font-variant-numeric: tabular-nums; } .billing-metric span { display: block; margin-top: 4px; color: var(--muted-foreground); font-size: 10px; letter-spacing: .06em; text-transform: uppercase; } .billing-note { grid-column: 1 / -1; margin: 0; padding: 11px 16px; border-top: 1px solid var(--border); color: var(--muted-foreground); font-size: 11px; } - .suite-nav { position: sticky; top: 0; z-index: 12; display: flex; flex-wrap: wrap; gap: 6px; margin: 0 0 24px; padding: 10px 0; border-top: 1px solid var(--border); border-bottom: 1px solid var(--border); background: var(--background); } - .suite-nav a { padding: 7px 10px; border-radius: calc(var(--radius) * .8); color: var(--muted-foreground); font-size: 12px; font-weight: 600; text-decoration: none; } - .suite-nav a:hover { background: var(--raised); color: var(--foreground); } + .results-heading { display: flex; align-items: baseline; justify-content: space-between; gap: 16px; margin: 40px 0 20px; } + .results-heading h2 { margin: 0; font-size: 24px; } + .results-heading a { color: var(--muted-foreground); font-size: 12px; } .suite-section, .history-section { scroll-margin-top: calc(var(--navigation-sticky-offset) + 16px); margin-top: 64px; } .section-heading { display: grid; grid-template-columns: minmax(0, 1fr) minmax(260px, 520px); align-items: end; gap: 32px; margin-bottom: 24px; } .section-heading h2 { margin: 0; font-size: clamp(25px, 3vw, 38px); line-height: 1.05; letter-spacing: -.035em; } @@ -913,6 +921,10 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .suite-summary span, .suite-summary small { color: var(--muted-foreground); font-size: 9px; } .suite-summary span { letter-spacing: .06em; text-transform: uppercase; } .suite-summary strong { margin: 3px 0 1px; font: 550 15px/1.3 var(--font-mono); } + .suite-case-toc { margin: 0 0 24px; } + .suite-case-toc-heading { display: flex; align-items: baseline; justify-content: space-between; gap: 16px; margin-bottom: 8px; } + .suite-case-toc-heading h3 { margin: 0; font-size: 13px; font-weight: 600; } + .suite-case-toc-heading > a { color: var(--muted-foreground); font-size: 11px; } .history-pointers { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 12px; margin-bottom: 16px; } .history-pointers a { padding: 14px 16px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--card); text-decoration: none; } .history-pointers span, .history-pointers strong { display: block; } @@ -948,7 +960,16 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .history-failed { border-color: var(--fail-border); background: var(--fail-bg); color: var(--fail-text); } .history-empty { padding: 24px 0; color: var(--muted-foreground); text-align: center; } .case-context > summary, .usage > summary { width: fit-content; cursor: pointer; color: var(--foreground); font-size: 12px; font-weight: 600; } + .case-diagnostics { margin-top: 14px; padding-top: 12px; border-top: 1px solid var(--border); } + .case-diagnostics > summary { display: flex; align-items: baseline; flex-wrap: wrap; gap: 4px 8px; width: 100%; list-style: none; } + .case-diagnostics > summary::-webkit-details-marker { display: none; } + .case-diagnostics > summary::before { content: "›"; color: var(--muted-foreground); font-size: 16px; line-height: 1; } + .case-diagnostics[open] > summary::before { transform: rotate(90deg); } + .case-detail-summary { margin-left: auto; color: var(--muted-foreground); font: 400 11px/1.5 var(--font-mono); } + .case-capabilities { padding-left: 16px; color: var(--muted-foreground); font-size: 11px; } .case-outcome { margin: 12px 0; padding: 12px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--raised); font-size: 12px; } + .case-incomplete > .case-outcome { border-color: var(--missing-border); } + .case-failed > .case-outcome { border-color: var(--fail-border); } .case-outcome p { margin: 8px 0; } .failure-reason { white-space: pre-wrap; overflow-wrap: anywhere; font: 11px/1.6 var(--font-mono); } .conversation-details { margin: 16px 0; } @@ -1033,6 +1054,8 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .gallery-previous { left: 24px; } .gallery-next { right: 24px; } .gallery-position { color: #b8b8b5; font: 12px/1.4 var(--font-mono); font-variant-numeric: tabular-nums; } + .gallery-original { color: #fafafa; font-size: 12px; } + .back-to-top { position: fixed; right: max(24px, env(safe-area-inset-right)); bottom: max(24px, env(safe-area-inset-bottom)); z-index: 10; display: inline-flex; align-items: center; gap: 8px; min-height: 44px; padding: 10px 16px; border: 1px solid var(--border-strong); border-radius: 999px; background: var(--foreground); color: var(--background); font: 600 12px/1.4 var(--font-sans); text-decoration: none; } @media (max-width: 1180px) { .matrix { display: block; min-width: 0; } .matrix thead { display: none; } @@ -1063,8 +1086,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .trend-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } .history-filters { grid-template-columns: repeat(2, minmax(0, 1fr)); } .report-filters { grid-template-columns: repeat(2, minmax(0, 1fr)); } - .report-search, .filter-result { grid-column: 1 / -1; } - .suite-nav { top: 0; } + .report-search, .report-feature, .filter-result { grid-column: 1 / -1; } .suite-section, .history-section { scroll-margin-top: 72px; } } @media (max-width: 640px) { @@ -1085,11 +1107,9 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .gallery-control { width: 40px; height: 56px; } .gallery-previous { left: 8px; } .gallery-next { right: 8px; } - .suite-nav { top: 0; overflow-x: auto; flex-wrap: nowrap; } - .suite-nav a { white-space: nowrap; } .history-pointers, .trend-grid, .history-filters { grid-template-columns: 1fr; } .report-filters { grid-template-columns: 1fr; } - .report-search, .filter-result { grid-column: auto; } + .report-search, .report-feature, .filter-result { grid-column: auto; } .suite-summary { grid-template-columns: 1fr; } .suite-summary > div, .suite-summary > div:nth-child(even) { border-right: 0; border-bottom: 1px solid var(--border); } .suite-summary > div:last-child { border-bottom: 0; } @@ -1098,8 +1118,8 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { @media (prefers-reduced-motion: reduce) { *, *::before, *::after { scroll-behavior: auto !important; transition-duration: 100ms !important; } } - .public-summary { margin: 0 0 32px; padding: 12px; border: 1px solid var(--border); border-radius: 10px; background: var(--raised); } - .public-summary img { display: block; width: 100%; height: auto; border-radius: 6px; } + ${campaignSuiteSummaryStyles} + ${journeyFeatureStyles} @media print { .brand-bar, .gallery-launch, dialog { display: none; } main { width: 100%; margin: 0; } @@ -1108,7 +1128,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { } - +
@@ -1120,7 +1140,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {

Full-stack acceptance campaign

-

${html(input.title)}

+

${html(input.title)}

A browser-verified matrix of runner profiles, execution environments, and deterministic task contracts. Declared PNG screenshots and normalized results are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.

@@ -1133,7 +1153,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
- ${publicSummaryImageHref ? `
Runner E2E campaign status summary
` : ""} + ${renderCampaignSuiteSummary(input.campaign?.suites ?? [], catalog, "", true)}
${html(tokenLabel(campaignBilling.llm.inputTokens, campaignBilling.llm))}Input tokens
${html(tokenLabel(campaignBilling.llm.outputTokens, campaignBilling.llm))}Output tokens
@@ -1147,24 +1167,23 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
${campaignBilling.llm.runsWithReportedCost}/${campaignBilling.llm.runCount}Runs provider-priced

Model spend is the provider-reported subtotal; unpriced or unavailable runs are excluded, never counted as free. Daytona runtime is a public-list-price estimate from captured lease time and pinned resources, before credits, discounts, storage allowance, or invoice adjustments. Local execution has no external runtime meter.

- + + ${renderJourneyFeatureCoverage(catalog, entryById)}
+ + - - +

${suiteSections} ${historySection} -
Generated ${html(input.generatedAt)}${catalog.length} catalog executions · Declared screenshots and normalized results published
+
Generated ${html(input.generatedAt)}${catalog.length} selected executions · Declared screenshots and normalized results published
+
@@ -1192,10 +1211,26 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
+ Open original image ↗
`); + } + + const contentTypes: Record = { + ".json": "application/json", ".png": "image/png", ".woff2": "font/woff2", + ".txt": "text/plain; charset=utf-8", ".md": "text/plain; charset=utf-8", + ".log": "text/plain; charset=utf-8", ".xml": "application/xml", + }; + const server = createServer(async (request, response) => { + response.setHeader("Cache-Control", "no-store"); + try { + if (request.method !== "GET" && request.method !== "HEAD") { + response.writeHead(405).end("Read-only preview"); + return; + } + const pathname = new URL(request.url ?? "/", "http://localhost").pathname; + if (pathname === "/__preview/version") { + response.writeHead(200, { "Content-Type": "text/plain" }).end(version); + return; + } + if (pathname === "/" || pathname === "/runner-e2e") { + response.writeHead(302, { Location: routeRoot }).end(); + return; + } + let html: string | undefined; + let bytes: Buffer | undefined; + let extension = ".html"; + if (pathname === routeRoot || pathname === `${routeRoot}index.html`) { + html = renderRunnerHistoryIndex(localHistory); + } else if (pathname.startsWith(routeRoot)) { + const relative = safePath(decodeURIComponent(pathname.slice(routeRoot.length)).replace(/\/$/, "")); + const campaignMatch = /^campaigns\/([A-Za-z0-9._-]+)(?:\/(.*))?$/.exec(relative); + if (campaignMatch && campaigns.has(campaignMatch[1]!)) { + const campaignId = campaignMatch[1]!; + const artifact = campaignMatch[2]; + if (!artifact || artifact === "index.html" || artifact === "dashboard.html") { + if (!artifact && !pathname.endsWith("/")) { + response.writeHead(302, { Location: `${pathname}/` }).end(); + return; + } + html = await campaignHtml(campaignId); + } else if ((await manifestFor(campaignId)).files.some((file) => file.path === artifact)) { + extension = path.extname(artifact); + if (contentTypes[extension]) bytes = await publicFile(relative); + } + } else if (relative === "history.json") { + extension = ".json"; + bytes = Buffer.from(JSON.stringify(retainedHistory)); + } else if (relative === "assets/InterVariable.woff2" || relative === "assets/favicon-32x32.png") { + extension = path.extname(relative); + bytes = await publicFile(relative); + } + } + if (html !== undefined) bytes = Buffer.from(withReload(html)); + if (!bytes) { + response.writeHead(404, { "Content-Type": "text/plain" }).end("Preview page or public artifact not found"); + return; + } + response.writeHead(200, { "Content-Type": contentTypes[extension] ?? "text/html; charset=utf-8" }); + response.end(request.method === "HEAD" ? undefined : bytes); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + response.writeHead(502, { "Content-Type": "text/plain; charset=utf-8" }).end(message); + } + }); + return { server, campaignCount: campaigns.size, cacheRoot }; +} + +async function main() { + const args = process.argv.slice(2).filter((argument) => argument !== "--"); + const portIndex = args.indexOf("--port"); + const port = portIndex < 0 ? 4317 : Number(args[portIndex + 1]); + if (!Number.isInteger(port) || port < 0 || port > 65535) throw new Error("Invalid --port"); + const { server, campaignCount, cacheRoot } = await createRunnerReportPreview({ + offline: args.includes("--offline"), refresh: args.includes("--refresh"), + }); + server.listen(port, "127.0.0.1", () => { + const address = server.address(); + const origin = `http://127.0.0.1:${typeof address === "object" && address ? address.port : port}`; + console.log(`History: ${origin}${routeRoot}`); + console.log(`${campaignCount} real campaigns; reports and public evidence cache on first access.`); + console.log(`Cached data: ${cacheRoot}`); + console.log("Edit history-index.ts or dashboard.ts; Node restarts the preview and the browser reloads."); + }); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main(); diff --git a/tests/runner-e2e/report-catalog.test.ts b/tests/runner-e2e/report-catalog.test.ts index b12062829c0..5cd96f1e078 100644 --- a/tests/runner-e2e/report-catalog.test.ts +++ b/tests/runner-e2e/report-catalog.test.ts @@ -10,6 +10,7 @@ import { discoverReportCatalog } from "./report-catalog.js"; import { buildRunnerCampaign } from "./history.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; +import { journeyCapabilities } from "./journey-features.js"; import type { RunnerE2EResult } from "./types.js"; const temporary: string[] = []; @@ -58,6 +59,34 @@ function campaign(results: RunnerE2EResult[] = [result()]) { } describe("trusted report catalog discovery", () => { + it("shows only the campaign selection while retaining a missing selected result", () => { + const known = runnerMatrix.find((execution) => execution.id === "core-compatibility.legacy-codex.local.ask-question")!; + const missing = runnerMatrix.find((execution) => execution.id === "core-compatibility.legacy-codex.local.message-marker")!; + const page = renderRunnerE2EDashboard({ + title: "Narrow campaign", generatedAt: "2026-10-20T12:00:00Z", + expected: [known.id, missing.id], catalog: runnerMatrix, + entries: [{ result: { ...result(), executionId: known.id, suiteId: known.suite.id, + profileId: known.profile.id, environmentId: known.environment.id, caseId: known.task.id }, valid: true, errors: [] }], + }); + expect(page).toContain(`data-execution-id="${known.id}"`); + expect(page).toContain(`data-execution-id="${missing.id}"`); + expect(page).toContain('class="case case-missing"'); + expect(page).not.toContain('id="suite-first-task"'); + expect(page).not.toContain('data-report-profile="runner-copilot"'); + expect(page).not.toContain('class="suite-nav"'); + expect(page.match(/data-report-case\s/g)).toHaveLength(2); + expect(page).toContain('data-report-features="questions-and-approvals"'); + expect(page).toContain('title="questions-and-approvals/questions"'); + expect(page).toContain("No mapped journey"); + expect(page).toContain("0/1 passed · 1 missing"); + }); + + it("leaves an unfamiliar journey unmapped even when its case name matches", () => { + const known = runnerMatrix.find((execution) => execution.id === "core-compatibility.legacy-codex.local.ask-question")!; + expect(journeyCapabilities(known).map((capability) => capability.id)).toEqual(["questions-and-approvals/questions"]); + expect(journeyCapabilities({ ...known, suite: { ...known.suite, id: "branch-only" } })).toEqual([]); + }); + it("aggregates unknown suites and renders their cases, missing results, and screenshot gallery", () => { const summary = campaign(); expect(summary.suites).toEqual([ diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index 1a43b5d25c6..a03852dec49 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -319,10 +319,10 @@ describe("runner E2E report aggregation", () => { expect(dashboard).toContain("Runner Full-Stack E2E"); expect(dashboard).toContain(executionId); expect(dashboard).toContain("case-passed"); - expect(dashboard).toContain( + expect(dashboard).not.toContain( "core-compatibility.runner-acpx-codex.daytona.message-marker", ); - expect(dashboard).toContain("case-not-selected"); + expect(dashboard).not.toContain('class="case case-not-selected"'); expect(dashboard).toContain(" { "Declared screenshots and normalized results published", ); expect(dashboard).toContain("message_contains"); - expect(dashboard).toContain("Matchers and test context"); + expect(dashboard).toContain("Test details"); expect(dashboard).toContain("Scheduler"); expect(dashboard).toContain("Run duration"); expect(dashboard).toContain("100ms"); @@ -361,9 +361,9 @@ describe("runner E2E report aggregation", () => { expect(dashboard).toContain("data-report-profile"); expect(dashboard).toContain("data-report-environment"); expect(dashboard).toContain("data-report-status"); - expect(dashboard.indexOf('class="report-filters"')).toBeGreaterThan( - dashboard.indexOf('class="suite-nav"'), - ); + expect(dashboard).not.toContain('class="suite-nav"'); + expect(dashboard).toContain("data-report-feature"); + expect(dashboard).toContain("Feature coverage"); expect(dashboard).not.toContain(".report-filters { position: sticky"); expect(dashboard).toContain("table-layout: fixed"); expect(dashboard).toContain('class="profile-column"');