import { expect, test, type Page } from "@playwright/test"; import { runnerExecutionById, runnerMatrix } from "../runner-e2e/catalog.js"; import { renderRunnerE2EDashboard, type RunnerDashboardEntry, } from "../runner-e2e/dashboard.js"; import { buildRunnerCampaign } from "../runner-e2e/history.js"; import type { MatrixExecution, RunnerE2EResult } from "../runner-e2e/types.js"; const reportUrl = "http://runner-e2e.test/campaign/"; async function openReport(page: Page, report: string) { await page.route(`${reportUrl}**`, (route) => route.fulfill({ contentType: "text/html", body: report })); await page.goto(reportUrl); } const executionIds = [ "core-compatibility.legacy-codex.local.message-marker", "core-compatibility.legacy-codex.daytona.message-marker", "core-compatibility.legacy-claude.local.message-marker", "local-session-integrity.runner-acpx-codex.local.structured-question-restart-resume", ] as const; function resultFor( execution: MatrixExecution, status: RunnerE2EResult["status"], ): RunnerDashboardEntry { const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v1", executionId: execution.id, suiteId: execution.suite.id, attempt: 1, status, profileId: execution.profile.id, environmentId: execution.environment.id, caseId: execution.task.id, provider: execution.profile.provider, model: execution.profile.model, runtimeMode: execution.profile.expectedRuntimeMode, startedAt: "2026-09-05T12:00:00.000Z", finishedAt: "2026-09-05T12:00:12.000Z", durationMs: 12_000, runIds: [`run-${execution.id}`], usage: { inputTokens: 1_250, outputTokens: 75, cachedInputTokens: 500, }, matcherResults: [ { matcher: { kind: "message_contains", expected: "PAPERCLIP_E2E_OK" }, passed: status === "passed", detail: status === "passed" ? "matched" : "marker missing", }, ], screenshots: [ { id: "final-state", label: "Final visible task state", file: "final-state.png", }, ], cleanup: "passed", }; return { result, valid: status === "passed", errors: status === "passed" ? [] : ["fixture failure"], evidenceBaseHref: `evidence/${execution.id}/attempt-1`, evidenceFiles: ["final-state.png"], }; } test("filters execution previews and pages through the filtered screenshot gallery", async ({ page, }) => { const catalog = executionIds.map(runnerExecutionById); const entries = catalog.map((execution, index) => resultFor(execution, index === 1 ? "failed" : "passed"), ); await openReport( page, renderRunnerE2EDashboard({ title: "Runner E2E interaction fixture", generatedAt: "2026-09-05T12:01:00.000Z", expected: executionIds, catalog: runnerMatrix, entries, }), ); const filters = page .locator("[data-report-query]") .locator("..") .locator(".."); await expect(page.locator(".suite-nav")).toHaveCount(0); await expect(page.locator(".suite-section")).toHaveCount(2); await expect(page.locator('select[data-report-suite] option')).toHaveCount(3); await expect(page.locator('select[data-report-profile] option')).toHaveCount(4); await expect(page.locator("[data-report-result]")).toHaveText( "4 of 4 executions shown", ); await expect(page.locator("[data-gallery-item]")).toHaveCount(4); const passedCard = page.locator('[data-execution-id="core-compatibility.legacy-codex.local.message-marker"]'); const failedCard = page.locator('[data-execution-id="core-compatibility.legacy-codex.daytona.message-marker"]'); await expect(passedCard.locator(".status")).toHaveText("passed"); await expect(passedCard.locator(".case-outcome")).toBeHidden(); await expect(passedCard.locator(".billing-strip")).toBeHidden(); await expect(passedCard.locator(".execution-id")).toBeHidden(); await expect(passedCard.locator(".case-diagnostics > summary")).toContainText("1/1 checks · 12s"); await expect(failedCard.locator(":scope > .case-outcome")).toBeVisible(); expect( await filters.evaluate((element) => getComputedStyle(element).position), ).toBe("static"); await page.locator('.feature-coverage > summary').click(); await page.locator('.feature-coverage').getByRole('button', { name: 'Questions and approvals', exact: true }).click(); await expect(page.locator('select[data-report-feature]')).toHaveValue('questions-and-approvals'); await expect(page.locator('.feature-coverage')).not.toHaveAttribute('open'); await expect(page.locator('[data-report-result]')).toHaveText('1 of 4 executions shown'); await expect(page.locator('.suite-section:visible')).toHaveCount(1); await expect(page.locator('[data-gallery-open]')).toHaveText('View gallery · 1'); await page.setViewportSize({ width: 900, height: 900 }); await expect(page.locator('.execution-preview:visible')).toHaveCount(1); await page.setViewportSize({ width: 1280, height: 720 }); await page.locator('[data-report-reset]').click(); const widthBefore = await passedCard.evaluate((card) => card.getBoundingClientRect().width); await passedCard.locator(".case-diagnostics > summary").click(); const widthAfter = await passedCard.evaluate((card) => card.getBoundingClientRect().width); expect(widthAfter).toBe(widthBefore); await expect(passedCard.locator(".billing-strip")).toBeVisible(); await page.locator("[data-report-query]").fill("Legacy Claude"); await expect(page.locator("[data-report-result]")).toHaveText( "1 of 4 executions shown", ); await expect(page.locator("[data-gallery-open]")).toHaveText( "View gallery · 1", ); await page.locator("[data-report-reset]").click(); await expect(page.locator("[data-report-query]")).toBeFocused(); await page .locator("select[data-report-profile]") .selectOption("legacy-codex"); await expect(page.locator("[data-report-result]")).toHaveText( "2 of 4 executions shown", ); await expect(page.locator("[data-gallery-open]")).toHaveText( "View gallery · 2", ); await page.locator("[data-gallery-open]").click(); const dialog = page.locator("[data-gallery-dialog]"); await expect(dialog).toBeVisible(); await expect(dialog.locator("[data-gallery-profile]")).toHaveText( "Legacy Codex", ); await expect(dialog.locator("[data-gallery-environment]")).toHaveText( "Isolated local", ); await expect(dialog.locator("[data-gallery-position]")).toHaveText( "Image 1 of 2", ); await expect(dialog.locator("[data-gallery-original]")).toHaveAttribute("href", /legacy-codex\.local\.message-marker\/attempt-1\/final-state\.png$/); await page.keyboard.press("ArrowRight"); await expect(dialog.locator("[data-gallery-environment]")).toHaveText( "Daytona sandbox", ); await expect(dialog.locator("[data-gallery-status]")).toHaveText("failed"); await expect(dialog.locator("[data-gallery-position]")).toHaveText( "Image 2 of 2", ); await expect(dialog.locator("[data-gallery-original]")).toHaveAttribute("href", /legacy-codex\.daytona\.message-marker\/attempt-1\/final-state\.png$/); await page.keyboard.press("Escape"); await expect(dialog).not.toBeVisible(); await page.locator("[data-report-reset]").click(); await page.keyboard.press("/"); await expect(page.locator("[data-report-query]")).toBeFocused(); }); function navigationReport(ids: readonly string[], suppliedEntries?: readonly RunnerDashboardEntry[]) { const catalog = ids.map(runnerExecutionById); const entries = suppliedEntries ?? catalog.map((execution) => resultFor(execution, "passed")); const generatedAt = "2026-09-05T12:01:00.000Z"; return renderRunnerE2EDashboard({ title: "Runner E2E navigation fixture", generatedAt, expected: ids, catalog: runnerMatrix, entries, campaign: buildRunnerCampaign({ campaignId: "navigation-fixture", generatedAt, expected: ids, catalog, results: entries.map((entry) => entry.result), }), }); } test("suite links reveal filtered sections and scroll on repeated clicks", async ({ page }) => { const report = navigationReport(executionIds); await openReport(page, report); const core = page.locator("#suite-core-compatibility"); const coreHeading = core.getByRole("heading", { name: "Core Runner Compatibility", exact: true }); const coreLink = page.locator('[data-nav-suite-link="core-compatibility"]'); const backToTop = page.getByRole("link", { name: "Back to top", exact: true }); await expect(backToTop).toBeHidden(); const feature = page.locator("select[data-report-feature]"); for (let click = 0; click < 2; click += 1) { await feature.selectOption("questions-and-approvals"); await expect(core).toBeHidden(); await coreLink.click(); await expect(core).toBeVisible(); await expect(coreHeading).toBeInViewport(); await expect(feature).toHaveValue(""); await expect(page).toHaveURL(`${reportUrl}?view=all#suite-core-compatibility`); await expect(page.locator("[data-report-result]")).toHaveText("4 of 4 executions shown"); } await page.locator("select[data-report-profile]").selectOption("legacy-codex"); await coreLink.click(); await expect(coreHeading).toBeInViewport(); await expect(page.locator("select[data-report-profile]")).toHaveValue("legacy-codex"); await expect(backToTop).toBeVisible(); expect(await backToTop.evaluate((element) => getComputedStyle(element).position)).toBe("fixed"); await backToTop.focus(); await page.keyboard.press("Enter"); await expect.poll(() => page.evaluate(() => window.scrollY)).toBe(0); await expect(backToTop).toBeHidden(); await expect(page.locator(".report-header h1")).toBeFocused(); await expect(page.locator("select[data-report-profile]")).toHaveValue("legacy-codex"); await coreLink.click(); await page.locator('[data-nav-suite-link="local-session-integrity"]').click(); await expect(page.locator("#suite-local-session-integrity")).toBeVisible(); await feature.selectOption("questions-and-approvals"); await page.goBack(); await expect(coreHeading).toBeInViewport(); await expect(feature).toHaveValue(""); }); test("folds the suite tree, inspects configurations, and returns to the scrolling report", async ({ page }) => { const ids = [...executionIds, "core-compatibility.legacy-codex.local.ask-question"]; await openReport(page, navigationReport(ids)); const coreIndex = page.locator('[data-nav-suite="core-compatibility"]'); const coreLink = coreIndex.locator('[data-nav-suite-link]'); await expect(coreIndex).not.toHaveAttribute("open"); await coreLink.click(); await expect(coreIndex).toHaveAttribute("open"); await coreIndex.locator("summary").focus(); await page.keyboard.press("Enter"); await expect(coreIndex).not.toHaveAttribute("open"); await page.keyboard.press("Enter"); await expect(coreIndex).toHaveAttribute("open"); await expect(coreIndex.locator("[data-nav-case]")).toHaveCount(2); const sectionToc = page.locator("#suite-core-compatibility .suite-case-toc"); await expect(sectionToc.locator(".suite-case-list a")).toHaveCount(2); const basicLink = coreIndex.getByRole("link", { name: "Basic response", exact: true }); const profile = page.locator("select[data-report-profile]"); const feature = page.locator("select[data-report-feature]"); const crumbs = page.getByRole("navigation", { name: "Breadcrumb", exact: true }); const returnToCampaign = crumbs.getByRole("link", { name: "navigation-fixture", exact: true }); await profile.selectOption("legacy-claude"); const readingPosition = await page.evaluate(() => window.scrollY); await basicLink.click(); const claudeResult = page.locator('[data-execution-id="core-compatibility.legacy-claude.local.message-marker"]'); await expect(claudeResult).toBeVisible(); await expect(page.locator("[data-report-case]:visible")).toHaveCount(1); await expect(page.getByRole("button", { name: "Legacy Claude · Isolated local", exact: true })).toHaveAttribute("aria-pressed", "true"); await expect(page.locator(".report-filters")).toBeHidden(); expect(new URL(page.url()).searchParams.get("view")).toBe("case"); expect(new URL(page.url()).hash).toBe("#execution-core-compatibility.legacy-claude.local.message-marker"); await returnToCampaign.click(); await expect(profile).toHaveValue("legacy-claude"); await expect.poll(() => page.evaluate(() => window.scrollY)).toBe(readingPosition); await feature.selectOption("questions-and-approvals"); await basicLink.click(); await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.local.message-marker"]')).toBeVisible(); await expect(page.locator("[data-nav-configuration]:visible")).toHaveCount(3); await crumbs.getByRole("link", { name: "Core Runner Compatibility", exact: true }).click(); await expect(feature).toHaveValue(""); await expect(profile).toHaveValue(""); await expect(page.locator("[data-report-case]:visible")).toHaveCount(5); await sectionToc.getByRole("link", { name: "Ask mode question", exact: true }).click(); await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.local.ask-question"] .case-heading')).toBeInViewport(); await sectionToc.getByRole("link", { name: "Back to overview", exact: false }).click(); await expect(page.locator("#overview")).toBeInViewport(); await page.setViewportSize({ width: 390, height: 844 }); await expect(coreIndex).toBeHidden(); await page.getByRole("button", { name: "Open navigation", exact: true }).click(); await expect(basicLink).toBeVisible(); expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); await basicLink.click(); await expect(coreIndex).toBeHidden(); const daytona = page.getByRole("button", { name: "Legacy Codex · Daytona sandbox", exact: true }); await daytona.click(); await page.reload(); await expect(daytona).toHaveAttribute("aria-pressed", "true"); await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.daytona.message-marker"]')).toBeVisible(); await expect(page.locator("[data-report-case]:visible")).toHaveCount(1); await crumbs.getByRole("link", { name: "Core Runner Compatibility", exact: true }).click(); const backToTop = page.getByRole("link", { name: "Back to top", exact: true }); await expect(backToTop).toBeVisible(); expect(await backToTop.evaluate((element) => { const bounds = element.getBoundingClientRect(); return bounds.left >= 0 && bounds.right <= innerWidth && bounds.bottom <= innerHeight; })).toBe(true); await backToTop.click(); await expect.poll(() => page.evaluate(() => window.scrollY)).toBe(0); }); test("case status dots summarize matching results and open failures", async ({ page }) => { const ids = [...executionIds, "core-compatibility.legacy-codex.local.ask-question"]; const entries = executionIds.map((id, index) => resultFor(runnerExecutionById(id), index === 1 ? "failed" : "passed")); await openReport(page, navigationReport(ids, entries)); const toc = page.locator("#suite-core-compatibility .suite-case-toc"); const basic = toc.getByRole("link", { name: "Basic response", exact: true }); const missing = toc.getByRole("link", { name: "Ask mode question", exact: true }); const topLink = page.locator('.report-suite-tree [data-report-case-link="core-compatibility.message-marker"]'); await expect(basic).toHaveAttribute("data-case-status", "failed"); await expect(topLink).toHaveAttribute("data-case-status", "failed"); await expect(basic).toHaveAccessibleDescription("failed · 2/3 configurations passed"); await expect(missing).toHaveAttribute("data-case-status", "missing"); await expect(missing).toHaveAttribute("title", "missing · 0/1 configurations passed"); const red = await basic.locator(".case-status-dot").evaluate((element) => getComputedStyle(element).backgroundColor); const profile = page.locator("select[data-report-profile]"); await profile.selectOption("legacy-claude"); await expect(basic).toHaveAttribute("data-case-status", "passed"); await expect(topLink).toHaveAttribute("data-case-status", "passed"); await expect(basic).toHaveAccessibleDescription("passed · 1/1 configurations passed"); const green = await basic.locator(".case-status-dot").evaluate((element) => getComputedStyle(element).backgroundColor); expect(green).not.toBe(red); await expect(missing).toHaveAttribute("title", /no matching configurations/); await page.locator('[data-report-journey="core-compatibility.message-marker"]').getByRole("link", { name: "Inspect case ↗", exact: true }).click(); await expect(page.locator('[data-execution-id="core-compatibility.legacy-claude.local.message-marker"]')).toBeVisible(); await expect(page.locator("[data-report-case]:visible")).toHaveCount(1); await expect(topLink).toHaveAttribute("data-case-status", "passed"); await page.getByRole("navigation", { name: "Breadcrumb", exact: true }).getByRole("link", { name: "navigation-fixture", exact: true }).click(); await profile.selectOption("legacy-codex"); await expect(basic).toHaveAttribute("data-case-status", "failed"); await page.locator("select[data-report-environment]").selectOption("local"); await expect(basic).toHaveAttribute("data-case-status", "passed"); await page.locator("[data-report-reset]").click(); await basic.click(); await expect(page).toHaveURL(`${reportUrl}?view=all#execution-core-compatibility.legacy-codex.daytona.message-marker`); await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.daytona.message-marker"] .case-heading')).toBeInViewport(); await page.locator('[data-execution-id="core-compatibility.legacy-codex.local.message-marker"] .case-open').click(); await expect(page.locator("[data-report-case]:visible")).toHaveCount(1); await expect(topLink).toHaveAttribute("data-case-status", "failed"); await expect(topLink).toHaveAccessibleDescription("failed · 2/3 configurations passed"); await page.getByRole("button", { name: "Legacy Codex · Daytona sandbox", exact: true }).click(); await expect(topLink).toHaveAttribute("data-case-status", "failed"); });