PaperClipAI/tests/e2e/runner-e2e-dashboard.spec.ts

import { expect, test, type Page } from "@playwright/test";

import { runnerExecutionById, runnerMatrix } from "../runner-e2e/catalog.js";
import {
  renderRunnerE2EDashboard,
  type RunnerDashboardEntry,
} from "../runner-e2e/dashboard.js";
import { buildRunnerCampaign } from "../runner-e2e/history.js";
import type { MatrixExecution, RunnerE2EResult } from "../runner-e2e/types.js";
import { projectTaskChatThread } from "../runner-e2e/task-chat.js";

const reportUrl = "http://runner-e2e.test/campaign/";

async function openReport(page: Page, report: string) {
  await page.route(`${reportUrl}**`, (route) => route.fulfill({ contentType: "text/html", body: report }));
  await page.goto(reportUrl);
}

const executionIds = [
  "core-compatibility.legacy-codex.local.message-marker",
  "core-compatibility.legacy-codex.daytona.message-marker",
  "core-compatibility.legacy-claude.local.message-marker",
  "local-session-integrity.runner-acpx-codex.local.structured-question-restart-resume",
] as const;

function resultFor(
  execution: MatrixExecution,
  status: RunnerE2EResult["status"],
): RunnerDashboardEntry {
  const result: RunnerE2EResult = {
    schema: "paperclip.runner-e2e.result/v1",
    executionId: execution.id,
    suiteId: execution.suite.id,
    attempt: 1,
    status,
    profileId: execution.profile.id,
    environmentId: execution.environment.id,
    caseId: execution.task.id,
    provider: execution.profile.provider,
    model: execution.profile.model,
    runtimeMode: execution.profile.expectedRuntimeMode,
    startedAt: "2026-09-05T12:00:00.000Z",
    finishedAt: "2026-09-05T12:00:12.000Z",
    durationMs: 12_000,
    runIds: [`run-${execution.id}`],
    usage: {
      inputTokens: 1_250,
      outputTokens: 75,
      cachedInputTokens: 500,
    },
    matcherResults: [
      {
        matcher: { kind: "message_contains", expected: "PAPERCLIP_E2E_OK" },
        passed: status === "passed",
        detail: status === "passed" ? "matched" : "marker missing",
      },
    ],
    screenshots: [
      {
        id: "final-state",
        label: "Final visible task state",
        file: "final-state.png",
      },
    ],
    cleanup: "passed",
  };

  return {
    result,
    valid: status === "passed",
    errors: status === "passed" ? [] : ["fixture failure"],
    evidenceBaseHref: `evidence/${execution.id}/attempt-1`,
    evidenceFiles: ["final-state.png"],
  };
}

test("expanding test details exposes the complete saved task conversation", async ({ page }) => {
  const execution = runnerExecutionById("core-compatibility.runner-codex.local.message-marker");
  const entry = resultFor(execution, "passed");
  const longReply = "Here is the result.\n\n" + "Retained paragraph. ".repeat(200) + "END OF COMPLETE REPLY";
  entry.result.taskChat = { schema: "paperclip.runner-e2e.task-chat/v1", capturedAt: "2026-10-28T12:01:00Z",
    status: "complete", missing: [], threads: [projectTaskChatThread(
      { id: "task", title: "Find the saved artifact", identifier: "E2E-1", description: "Find our saved report.", createdAt: "2026-10-28T12:00:00Z" },
      [{ id: "request", body: "Please include the download link.", createdAt: "2026-10-28T12:00:01Z" },
        { id: "reply", authorAgentId: "agent", body: longReply, createdAt: "2026-10-28T12:00:02Z" }], [], new Map([["agent", "Morgan"]]))] };
  await openReport(page, renderRunnerE2EDashboard({ title: "Task chat recording", catalog: [execution], entries: [entry], expected: [execution.id], generatedAt: "2026-10-28T12:01:00Z" }));
  const chat = page.getByRole("region", { name: "Task chat", exact: true });
  await expect(chat).not.toBeVisible();
  await page.getByText("Test details", { exact: true }).click();
  await expect(chat).toBeVisible();
  await expect(chat.getByText("Find our saved report.", { exact: true })).toBeVisible();
  await expect(chat.getByText("Please include the download link.", { exact: true })).toBeVisible();
  await expect(chat.getByText("Morgan", { exact: true })).toBeVisible();
  await expect(chat.locator(".transcript-text").last()).toHaveText(longReply);
  await expect(chat.getByText(/END OF COMPLETE REPLY/)).toBeVisible();
});

test("filters execution previews and pages through the filtered screenshot gallery", async ({
  page,
}) => {
  const catalog = executionIds.map(runnerExecutionById);
  const entries = catalog.map((execution, index) =>
    resultFor(execution, index === 1 ? "failed" : "passed"),
  );
  await openReport(
    page,
    renderRunnerE2EDashboard({
      title: "Runner E2E interaction fixture",
      generatedAt: "2026-09-05T12:01:00.000Z",
      expected: executionIds,
      catalog: runnerMatrix,
      entries,
    }),
  );

  const filters = page
    .locator("[data-report-query]")
    .locator("..")
    .locator("..");
  await expect(page.locator(".suite-nav")).toHaveCount(0);
  await expect(page.locator(".suite-section")).toHaveCount(2);
  await expect(page.locator('select[data-report-suite] option')).toHaveCount(3);
  await expect(page.locator('select[data-report-profile] option')).toHaveCount(4);
  await expect(page.locator("[data-report-result]")).toHaveText(
    "4 of 4 executions shown",
  );
  await expect(page.locator("[data-gallery-item]")).toHaveCount(4);
  const passedCard = page.locator('[data-execution-id="core-compatibility.legacy-codex.local.message-marker"]');
  const failedCard = page.locator('[data-execution-id="core-compatibility.legacy-codex.daytona.message-marker"]');
  await expect(passedCard.locator(".status")).toHaveText("passed");
  await expect(passedCard.locator(".case-outcome")).toBeHidden();
  await expect(passedCard.locator(".billing-strip")).toBeHidden();
  await expect(passedCard.locator(".execution-id")).toBeHidden();
  await expect(passedCard.locator(".case-diagnostics > summary")).toContainText("1/1 checks · 12s");
  await expect(failedCard.locator(":scope > .case-outcome")).toBeVisible();
  expect(
    await filters.evaluate((element) => getComputedStyle(element).position),
  ).toBe("static");
  await page.locator('.feature-coverage > summary').click();
  await page.locator('.feature-coverage').getByRole('button', { name: 'Questions and approvals', exact: true }).click();
  await expect(page.locator('select[data-report-feature]')).toHaveValue('questions-and-approvals');
  await expect(page.locator('.feature-coverage')).not.toHaveAttribute('open');
  await expect(page.locator('[data-report-result]')).toHaveText('1 of 4 executions shown');
  await expect(page.locator('.suite-section:visible')).toHaveCount(1);
  await expect(page.locator('[data-gallery-open]')).toHaveText('View gallery · 1');
  await page.setViewportSize({ width: 900, height: 900 });
  await expect(page.locator('.execution-preview:visible')).toHaveCount(1);
  await page.setViewportSize({ width: 1280, height: 720 });
  await page.locator('[data-report-reset]').click();

  const widthBefore = await passedCard.evaluate((card) => card.getBoundingClientRect().width);
  await passedCard.locator(".case-diagnostics > summary").click();
  const widthAfter = await passedCard.evaluate((card) => card.getBoundingClientRect().width);
  expect(widthAfter).toBe(widthBefore);
  await expect(passedCard.locator(".billing-strip")).toBeVisible();

  await page.locator("[data-report-query]").fill("Legacy Claude");
  await expect(page.locator("[data-report-result]")).toHaveText(
    "1 of 4 executions shown",
  );
  await expect(page.locator("[data-gallery-open]")).toHaveText(
    "View gallery · 1",
  );

  await page.locator("[data-report-reset]").click();
  await expect(page.locator("[data-report-query]")).toBeFocused();
  await page
    .locator("select[data-report-profile]")
    .selectOption("legacy-codex");
  await expect(page.locator("[data-report-result]")).toHaveText(
    "2 of 4 executions shown",
  );
  await expect(page.locator("[data-gallery-open]")).toHaveText(
    "View gallery · 2",
  );

  await page.locator("[data-gallery-open]").click();
  const dialog = page.locator("[data-gallery-dialog]");
  await expect(dialog).toBeVisible();
  await expect(dialog.locator("[data-gallery-profile]")).toHaveText(
    "Legacy Codex",
  );
  await expect(dialog.locator("[data-gallery-environment]")).toHaveText(
    "Isolated local",
  );
  await expect(dialog.locator("[data-gallery-position]")).toHaveText(
    "Image 1 of 2",
  );
  await expect(dialog.locator("[data-gallery-original]")).toHaveAttribute("href", /legacy-codex\.local\.message-marker\/attempt-1\/final-state\.png$/);

  await page.keyboard.press("ArrowRight");
  await expect(dialog.locator("[data-gallery-environment]")).toHaveText(
    "Daytona sandbox",
  );
  await expect(dialog.locator("[data-gallery-status]")).toHaveText("failed");
  await expect(dialog.locator("[data-gallery-position]")).toHaveText(
    "Image 2 of 2",
  );
  await expect(dialog.locator("[data-gallery-original]")).toHaveAttribute("href", /legacy-codex\.daytona\.message-marker\/attempt-1\/final-state\.png$/);

  await page.keyboard.press("Escape");
  await expect(dialog).not.toBeVisible();
  await page.locator("[data-report-reset]").click();
  await page.keyboard.press("/");
  await expect(page.locator("[data-report-query]")).toBeFocused();
});

function navigationReport(ids: readonly string[], suppliedEntries?: readonly RunnerDashboardEntry[]) {
  const catalog = ids.map(runnerExecutionById);
  const entries = suppliedEntries ?? catalog.map((execution) => resultFor(execution, "passed"));
  const generatedAt = "2026-09-05T12:01:00.000Z";
  return renderRunnerE2EDashboard({
    title: "Runner E2E navigation fixture",
    generatedAt,
    expected: ids,
    catalog: runnerMatrix,
    entries,
    campaign: buildRunnerCampaign({
      campaignId: "navigation-fixture",
      generatedAt,
      expected: ids,
      catalog,
      results: entries.map((entry) => entry.result),
    }),
  });
}

test("suite links reveal filtered sections and scroll on repeated clicks", async ({ page }) => {
  const report = navigationReport(executionIds);
  await openReport(page, report);

  const core = page.locator("#suite-core-compatibility");
  const coreHeading = core.getByRole("heading", { name: "Core Runner Compatibility", exact: true });
  const coreLink = page.locator('[data-nav-suite-link="core-compatibility"]');
  const backToTop = page.getByRole("link", { name: "Back to top", exact: true });
  await expect(backToTop).toBeHidden();
  const feature = page.locator("select[data-report-feature]");
  for (let click = 0; click < 2; click += 1) {
    await feature.selectOption("questions-and-approvals");
    await expect(core).toBeHidden();
    await coreLink.click();
    await expect(core).toBeVisible();
    await expect(coreHeading).toBeInViewport();
    await expect(feature).toHaveValue("");
    await expect(page).toHaveURL(`${reportUrl}?view=all#suite-core-compatibility`);
    await expect(page.locator("[data-report-result]")).toHaveText("4 of 4 executions shown");
  }

  await page.locator("select[data-report-profile]").selectOption("legacy-codex");
  await coreLink.click();
  await expect(coreHeading).toBeInViewport();
  await expect(page.locator("select[data-report-profile]")).toHaveValue("legacy-codex");
  await expect(backToTop).toBeVisible();
  expect(await backToTop.evaluate((element) => getComputedStyle(element).position)).toBe("fixed");
  await backToTop.focus();
  await page.keyboard.press("Enter");
  await expect.poll(() => page.evaluate(() => window.scrollY)).toBe(0);
  await expect(backToTop).toBeHidden();
  await expect(page.locator(".report-header h1")).toBeFocused();
  await expect(page.locator("select[data-report-profile]")).toHaveValue("legacy-codex");
  await coreLink.click();

  await page.locator('[data-nav-suite-link="local-session-integrity"]').click();
  await expect(page.locator("#suite-local-session-integrity")).toBeVisible();
  await feature.selectOption("questions-and-approvals");
  await page.goBack();
  await expect(coreHeading).toBeInViewport();
  await expect(feature).toHaveValue("");
});

test("folds the suite tree, inspects configurations, and returns to the scrolling report", async ({ page }) => {
  const ids = [...executionIds, "core-compatibility.legacy-codex.local.ask-question"];
  await openReport(page, navigationReport(ids));
  const coreIndex = page.locator('[data-nav-suite="core-compatibility"]');
  const coreLink = coreIndex.locator('[data-nav-suite-link]');
  await expect(coreIndex).not.toHaveAttribute("open");
  await coreLink.click();
  await expect(coreIndex).toHaveAttribute("open");
  await coreIndex.locator("summary").focus();
  await page.keyboard.press("Enter");
  await expect(coreIndex).not.toHaveAttribute("open");
  await page.keyboard.press("Enter");
  await expect(coreIndex).toHaveAttribute("open");
  await expect(coreIndex.locator("[data-nav-case]")).toHaveCount(2);
  const sectionToc = page.locator("#suite-core-compatibility .suite-case-toc");
  await expect(sectionToc.locator(".suite-case-list a")).toHaveCount(2);

  const basicLink = coreIndex.getByRole("link", { name: "Basic response", exact: true });
  const profile = page.locator("select[data-report-profile]");
  const feature = page.locator("select[data-report-feature]");
  const crumbs = page.getByRole("navigation", { name: "Breadcrumb", exact: true });
  const returnToCampaign = crumbs.getByRole("link", { name: "navigation-fixture", exact: true });
  await profile.selectOption("legacy-claude");
  const readingPosition = await page.evaluate(() => window.scrollY);
  await basicLink.click();
  const claudeResult = page.locator('[data-execution-id="core-compatibility.legacy-claude.local.message-marker"]');
  await expect(claudeResult).toBeVisible();
  await expect(page.locator("[data-report-case]:visible")).toHaveCount(1);
  await expect(page.getByRole("button", { name: "Legacy Claude · Isolated local", exact: true })).toHaveAttribute("aria-pressed", "true");
  await expect(page.locator(".report-filters")).toBeHidden();
  expect(new URL(page.url()).searchParams.get("view")).toBe("case");
  expect(new URL(page.url()).hash).toBe("#execution-core-compatibility.legacy-claude.local.message-marker");
  await returnToCampaign.click();
  await expect(profile).toHaveValue("legacy-claude");
  await expect.poll(() => page.evaluate(() => window.scrollY)).toBe(readingPosition);

  await feature.selectOption("questions-and-approvals");
  await basicLink.click();
  await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.local.message-marker"]')).toBeVisible();
  await expect(page.locator("[data-nav-configuration]:visible")).toHaveCount(3);
  await crumbs.getByRole("link", { name: "Core Runner Compatibility", exact: true }).click();
  await expect(feature).toHaveValue("");
  await expect(profile).toHaveValue("");
  await expect(page.locator("[data-report-case]:visible")).toHaveCount(5);
  await sectionToc.getByRole("link", { name: "Ask mode question", exact: true }).click();
  await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.local.ask-question"] .case-heading')).toBeInViewport();
  await sectionToc.getByRole("link", { name: "Back to overview", exact: false }).click();
  await expect(page.locator("#overview")).toBeInViewport();

  await page.setViewportSize({ width: 390, height: 844 });
  await expect(coreIndex).toBeHidden();
  await page.getByRole("button", { name: "Open navigation", exact: true }).click();
  await expect(basicLink).toBeVisible();
  expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true);
  await basicLink.click();
  await expect(coreIndex).toBeHidden();
  const daytona = page.getByRole("button", { name: "Legacy Codex · Daytona sandbox", exact: true });
  await daytona.click();
  await page.reload();
  await expect(daytona).toHaveAttribute("aria-pressed", "true");
  await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.daytona.message-marker"]')).toBeVisible();
  await expect(page.locator("[data-report-case]:visible")).toHaveCount(1);
  await crumbs.getByRole("link", { name: "Core Runner Compatibility", exact: true }).click();
  const backToTop = page.getByRole("link", { name: "Back to top", exact: true });
  await expect(backToTop).toBeVisible();
  expect(await backToTop.evaluate((element) => {
    const bounds = element.getBoundingClientRect();
    return bounds.left >= 0 && bounds.right <= innerWidth && bounds.bottom <= innerHeight;
  })).toBe(true);
  await backToTop.click();
  await expect.poll(() => page.evaluate(() => window.scrollY)).toBe(0);
});

test("case status dots summarize matching results and open failures", async ({ page }) => {
  const ids = [...executionIds, "core-compatibility.legacy-codex.local.ask-question"];
  const entries = executionIds.map((id, index) => resultFor(runnerExecutionById(id), index === 1 ? "failed" : "passed"));
  await openReport(page, navigationReport(ids, entries));
  const toc = page.locator("#suite-core-compatibility .suite-case-toc");
  const basic = toc.getByRole("link", { name: "Basic response", exact: true });
  const missing = toc.getByRole("link", { name: "Ask mode question", exact: true });
  const topLink = page.locator('.report-suite-tree [data-report-case-link="core-compatibility.message-marker"]');
  await expect(basic).toHaveAttribute("data-case-status", "failed");
  await expect(topLink).toHaveAttribute("data-case-status", "failed");
  await expect(basic).toHaveAccessibleDescription("failed · 2/3 configurations passed");
  await expect(missing).toHaveAttribute("data-case-status", "missing");
  await expect(missing).toHaveAttribute("title", "missing · 0/1 configurations passed");
  const red = await basic.locator(".case-status-dot").evaluate((element) => getComputedStyle(element).backgroundColor);

  const profile = page.locator("select[data-report-profile]");
  await profile.selectOption("legacy-claude");
  await expect(basic).toHaveAttribute("data-case-status", "passed");
  await expect(topLink).toHaveAttribute("data-case-status", "passed");
  await expect(basic).toHaveAccessibleDescription("passed · 1/1 configurations passed");
  const green = await basic.locator(".case-status-dot").evaluate((element) => getComputedStyle(element).backgroundColor);
  expect(green).not.toBe(red);
  await expect(missing).toHaveAttribute("title", /no matching configurations/);
  await page.locator('[data-report-journey="core-compatibility.message-marker"]').getByRole("link", { name: "Inspect case ↗", exact: true }).click();
  await expect(page.locator('[data-execution-id="core-compatibility.legacy-claude.local.message-marker"]')).toBeVisible();
  await expect(page.locator("[data-report-case]:visible")).toHaveCount(1);
  await expect(topLink).toHaveAttribute("data-case-status", "passed");
  await page.getByRole("navigation", { name: "Breadcrumb", exact: true }).getByRole("link", { name: "navigation-fixture", exact: true }).click();

  await profile.selectOption("legacy-codex");
  await expect(basic).toHaveAttribute("data-case-status", "failed");
  await page.locator("select[data-report-environment]").selectOption("local");
  await expect(basic).toHaveAttribute("data-case-status", "passed");
  await page.locator("[data-report-reset]").click();
  await basic.click();
  await expect(page).toHaveURL(`${reportUrl}?view=all#execution-core-compatibility.legacy-codex.daytona.message-marker`);
  await expect(page.locator('[data-execution-id="core-compatibility.legacy-codex.daytona.message-marker"] .case-heading')).toBeInViewport();
  await page.locator('[data-execution-id="core-compatibility.legacy-codex.local.message-marker"] .case-open').click();
  await expect(page.locator("[data-report-case]:visible")).toHaveCount(1);
  await expect(topLink).toHaveAttribute("data-case-status", "failed");
  await expect(topLink).toHaveAccessibleDescription("failed · 2/3 configurations passed");
  await page.getByRole("button", { name: "Legacy Codex · Daytona sandbox", exact: true }).click();
  await expect(topLink).toHaveAttribute("data-case-status", "failed");
});

test("keeps long feedback fingerprints inside a scrollable review table on phones", async ({ page }) => {
  const { emptyFeedbackSummary } = await import("../runner-e2e/feedback.js");
  const { feedbackReviewTemplate } = await import("../runner-e2e/feedback-dispositions.js");
  const execution = runnerExecutionById(executionIds[0]);
  const entry = resultFor(execution, "passed");
  entry.result.feedback = { ...emptyFeedbackSummary(), collection: "complete", invocationCollection: "complete",
    complaints: 2, bodiesCollected: 2,
    review: ["a", "b"].map(value => ({ fingerprint: value.repeat(64), kind: "complaint", occurrences: 1, disposition: "needs_investigation" })),
  };
  const campaign = buildRunnerCampaign({ campaignId: "feedback-mobile", generatedAt: entry.result.finishedAt, expected: [execution.id], results: [entry.result] });
  campaign.feedbackReview = { ...feedbackReviewTemplate(campaign), version: 1, reviewedAt: "2026-10-20T01:00:00Z", reviewedBy: "agent",
    decisions: [{ fingerprint: "a".repeat(64), disposition: "actionable", issueNumber: 123 },
      { fingerprint: "b".repeat(64), disposition: "duplicate", duplicateOf: "a".repeat(64) }],
  };
  await page.setViewportSize({ width: 390, height: 844 });
  await openReport(page, renderRunnerE2EDashboard({ title: "Feedback review", generatedAt: campaign.generatedAt,
    expected: campaign.expected, catalog: [execution], entries: [entry], campaign }));
  await page.getByText("Campaign usage and spend", { exact: true }).click();
  const decisions = page.getByRole("region", { name: "Feedback review decisions" });
  await expect(decisions).toBeVisible();
  expect(await decisions.evaluate(element => element.scrollWidth > element.clientWidth)).toBe(true);
  expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(390);
  await decisions.focus();
  await page.keyboard.press("Tab");
  const followup = decisions.getByRole("link", { name: "#123" });
  await expect(followup).toBeFocused();
  await expect(followup).toBeInViewport();
});