PaperClipAI/tests/runner-e2e/report.test.ts

import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import type { RunnerE2EResult } from "./types.js";

const execFileAsync = promisify(execFile);
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
const cleanupDirectories: string[] = [];
afterEach(async () => {
  await Promise.all(
    cleanupDirectories
      .splice(0)
      .map((directory) => rm(directory, { recursive: true, force: true })),
  );
});

describe("runner E2E report aggregation", () => {
  it("keeps interrupted journeys incomplete unless their evidence is invalid", async () => {
    const root = await mkdtemp(path.join(os.tmpdir(), "runner-incomplete-report-"));
    cleanupDirectories.push(root);
    const ids: string[] = [];
    for (const [caseId, validEvidence] of [["task-card-accept", true], ["task-reply-accept", false]] as const) {
      const executionId = `first-task.runner-codex.local.${caseId}`;
      ids.push(executionId);
      const directory = path.join(root, caseId);
      await mkdir(directory);
      await writeFile(path.join(directory, "result.json"), JSON.stringify({
        schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "first-task",
        attempt: 1, status: "failed", failureClass: "candidate_failure", error: "Recording stopped before acceptance",
        profileId: "runner-codex", environmentId: "local", caseId, provider: "codex", model: "fixture-model", runtimeMode: "native",
        startedAt: "2026-09-15T00:00:00Z", finishedAt: "2026-09-15T00:01:00Z", durationMs: 60_000, cleanup: "passed",
        firstTask: { caseId, nonce: "fixture", onboardingIssueId: "task", agentId: "agent", initialTaskIds: ["task"], instructions: [], configuredModel: null, observedModels: [], checkpoints: [],
          checks: [{ id: "acceptance-recorded", passed: false, notReached: "No acceptance checkpoint", detail: "Acceptance recorded", evidence: [] }] },
      } satisfies RunnerE2EResult));
      if (validEvidence) await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: [], leaks: [], missing: [] }));
    }
    const output = path.join(root, "merged");
    await expect(execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], {
      cwd: repositoryRoot, env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify(ids) },
    })).rejects.toBeDefined();
    const normalized = JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8"));
    expect(normalized).toMatchObject({ passed: 0, failed: 1, incomplete: 1 });
    expect(normalized.results[0]).toMatchObject({ evidenceValid: true, evidenceErrors: [] });
    expect(normalized.results[1]).toMatchObject({ evidenceValid: false, failureClass: "permanent_infrastructure" });
    const page = await readFile(path.join(output, "index.html"), "utf8");
    expect(page).toContain("Incomplete journey");
    expect(page).toContain("evidence manifest missing");
  });

  it("selects the latest retry and enforces cleanup and pass evidence", async () => {
    const root = await mkdtemp(
      path.join(os.tmpdir(), "runner-e2e-report-test-"),
    );
    cleanupDirectories.push(root);
    const executionId = "legacy-codex.local.message-marker";
    const base: RunnerE2EResult = {
      schema: "paperclip.runner-e2e.result/v1",
      executionId,
      attempt: 2,
      status: "passed",
      profileId: "legacy-codex",
      environmentId: "local",
      caseId: "message-marker",
      provider: "codex",
      model: "fixture-model",
      runtimeMode: "legacy",
      startedAt: "2026-08-26T00:00:00.000Z",
      finishedAt: "2026-08-26T00:00:01.000Z",
      durationMs: 1_000,
      source: {
        sha: "forged-result-sha",
        ref: "refs/heads/forged-result",
        workflowRunUrl: "https://example.test/actions/runs/forged",
      },
      runIds: ["run-2"],
      turnTimings: [
        {
          turn: 1,
          submittedAt: "2026-08-26T00:00:00.000Z",
          runStartedAt: "2026-08-26T00:00:00.100Z",
          runFinishedAt: "2026-08-26T00:00:01.000Z",
          schedulerLatencyMs: 100,
          runDurationMs: 900,
          responseLatencyMs: 1_000,
          runId: "run-2",
          leaseAcquisitionOutcome: "created",
        },
      ],
      usage: {
        inputTokens: 1_250,
        outputTokens: 75,
        cachedInputTokens: 500,
        costUsd: 0.0125,
      },
      cleanup: "passed",
    };
    for (const attempt of [1, 2]) {
      const directory = path.join(root, `attempt-${attempt}`);
      await mkdir(directory, { recursive: true });
      const result =
        attempt === 1
          ? {
              ...base,
              attempt,
              status: "failed" as const,
              failureClass: "transient_infrastructure" as const,
            }
          : {
              ...base,
              matcherResults: [
                {
                  matcher: {
                    kind: "message_contains" as const,
                    expected: "PAPERCLIP_E2E_OK",
                  },
                  passed: true,
                  detail: "matched",
                },
              ],
              screenshots: [
                {
                  id: "final-state",
                  label: "Final visible task state",
                  file: "final-state.png",
                },
              ],
            };
      await writeFile(
        path.join(directory, "result.json"),
        JSON.stringify(result),
      );
      if (attempt === 2) {
        await writeFile(path.join(directory, "final-state.png"), "fake-png");
      }
      await writeFile(
        path.join(directory, "evidence-manifest.json"),
        JSON.stringify({
          files: attempt === 2 ? ["final-state.png"] : [],
          leaks: [],
          missing: [],
        }),
      );
    }
    const staleDuplicate = path.join(root, "attempt-2-stale-duplicate");
    await mkdir(staleDuplicate, { recursive: true });
    await writeFile(
      path.join(staleDuplicate, "result.json"),
      JSON.stringify({
        ...base,
        status: "failed",
        failureClass: "candidate_failure",
        finishedAt: "2026-08-26T00:00:00.500Z",
      }),
    );
    await writeFile(
      path.join(staleDuplicate, "evidence-manifest.json"),
      JSON.stringify({ files: [], leaks: [], missing: [] }),
    );
    const output = path.join(root, "merged");
    await execFileAsync(
      process.execPath,
      [
        path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"),
        path.join(repositoryRoot, "tests/runner-e2e/report.ts"),
      ],
      {
        cwd: repositoryRoot,
        env: {
          ...process.env,
          PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root,
          PAPERCLIP_RUNNER_E2E_REPORT_OUT: output,
          PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]),
          PAPERCLIP_RUNNER_E2E_SOURCE_SHA:
            "0123456789abcdef0123456789abcdef01234567",
          PAPERCLIP_RUNNER_E2E_SOURCE_REF:
            "refs/heads/fix/runner-paid-source-attribution",
          GITHUB_SHA: "trusted-default-workflow-sha",
          GITHUB_REF: "refs/heads/master",
          GITHUB_SERVER_URL: "https://github.com",
          GITHUB_REPOSITORY: "paperclipai/paperclip",
          GITHUB_RUN_ID: "123456",
          PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL:
            "https://reports.example.test/",
          PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: "/runner-e2e/",
        },
      },
    );
    const normalized = JSON.parse(
      await readFile(path.join(output, "normalized-results.json"), "utf8"),
    );
    expect(normalized).toMatchObject({
      schema: "paperclip.runner-e2e.campaign/v2",
      selected: 1,
      executed: 1,
      passed: 1,
      failed: 0,
      retries: 1,
      cleanupPassed: true,
      source: {
        sha: "0123456789abcdef0123456789abcdef01234567",
        ref: "refs/heads/fix/runner-paid-source-attribution",
        workflowRunUrl:
          "https://github.com/paperclipai/paperclip/actions/runs/123456",
      },
    });
    expect(normalized.billing).toMatchObject({
      reportedLlmCostUsd: 0.0125,
      llm: {
        inputTokens: 1_250,
        outputTokens: 75,
        runsWithReportedCost: 1,
      },
    });
    expect(normalized.results[0]).toMatchObject({
      attempt: 2,
      evidenceValid: true,
      source: {
        sha: "0123456789abcdef0123456789abcdef01234567",
        ref: "refs/heads/fix/runner-paid-source-attribution",
        workflowRunUrl:
          "https://github.com/paperclipai/paperclip/actions/runs/123456",
      },
    });
    const dashboard = await readFile(
      path.join(output, "dashboard.html"),
      "utf8",
    );
    expect(dashboard).toContain("Runner Full-Stack E2E");
    expect(dashboard).toContain(executionId);
    expect(dashboard).toContain("case-passed");
    expect(dashboard).toContain(
      "core-compatibility.runner-acpx-codex.daytona.message-marker",
    );
    expect(dashboard).toContain("case-not-selected");
    expect(dashboard).toContain("<img");
    expect(dashboard).toContain('class="brand-lockup"');
    expect(dashboard).toContain("data-gallery-dialog");
    expect(dashboard).toContain(
      `id="execution-core-compatibility.${executionId}"`,
    );
    expect(dashboard).toContain("data-gallery-previous");
    expect(dashboard).toContain("data-gallery-next");
    expect(dashboard).toContain("View gallery · 1");
    expect(dashboard).toContain(
      "Declared PNG screenshots and sanitized structured evidence are retained with every published campaign",
    );
    expect(dashboard).toContain(
      "Declared screenshots and sanitized structured evidence published",
    );
    expect(dashboard).toContain("message_contains");
    expect(dashboard).toContain("Matchers and test context");
    expect(dashboard).toContain("Scheduler");
    expect(dashboard).toContain("Run duration");
    expect(dashboard).toContain("100ms");
    expect(dashboard).toContain("Campaign billing summary");
    expect(dashboard).toContain("LLM reported subtotal");
    expect(dashboard).toContain("Agent execution time");
    expect(dashboard).toContain("Daytona lease time");
    expect(dashboard).toContain("1,250 in · 75 out");
    expect(dashboard).toContain("$0.0125");
    expect(dashboard).toContain("unpriced or unavailable runs are excluded");
    expect(dashboard).toContain('class="profile-sticky"');
    expect(dashboard).toContain('class="mobile-environment-header"');
    expect(dashboard).toContain("data-gallery-profile=");
    expect(dashboard).toContain("data-gallery-environment=");
    expect(dashboard).toContain("data-gallery-duration=");
    expect(dashboard).toContain("data-gallery-tokens=");
    expect(dashboard).toContain("data-gallery-matchers=");
    expect(dashboard).toContain("data-report-query");
    expect(dashboard).toContain("data-report-profile");
    expect(dashboard).toContain("data-report-environment");
    expect(dashboard).toContain("data-report-status");
    expect(dashboard.indexOf('class="report-filters"')).toBeGreaterThan(
      dashboard.indexOf('class="suite-nav"'),
    );
    expect(dashboard).not.toContain(".report-filters { position: sticky");
    expect(dashboard).toContain("table-layout: fixed");
    expect(dashboard).toContain('class="profile-column"');
    expect(dashboard).toContain('aria-label="Previous"');
    expect(dashboard).toContain('aria-label="Next"');
    expect(dashboard).not.toContain("overflow: auto; max-height: calc(100vh");
    expect(dashboard).toContain("@media (max-width: 1180px)");
    expect(
      await readFile(path.join(output, "assets", "favicon-32x32.png")),
    ).not.toHaveLength(0);
    expect(
      await readFile(path.join(output, "assets", "InterVariable.woff2")),
    ).not.toHaveLength(0);
    expect(
      await readFile(
        path.join(
          output,
          "evidence",
          `core-compatibility.${executionId}`,
          "attempt-2",
          "final-state.png",
        ),
        "utf8",
      ),
    ).toBe("fake-png");
    expect(await readFile(path.join(output, "index.html"), "utf8")).toBe(
      dashboard,
    );
    const summary = await readFile(path.join(output, "summary.md"), "utf8");
    expect(summary).toContain("## View results");
    expect(summary).toContain(
      "[Open the exact interactive campaign report](https://reports.example.test/runner-e2e/campaigns/gha-123456-1/index.html)",
    );
    expect(summary).toContain(
      "[Open the workflow run and per-cell job logs](https://github.com/paperclipai/paperclip/actions/runs/123456)",
    );
    expect(summary).toContain(
      "[Download the merged report and per-cell evidence](https://github.com/paperclipai/paperclip/actions/runs/123456#artifacts)",
    );
    expect(summary).toContain(
      `[core-compatibility.${executionId}](https://reports.example.test/runner-e2e/campaigns/gha-123456-1/index.html#execution-core-compatibility.${executionId})`,
    );
  });

  it("prefers a valid rerun over a higher attempt number from an older campaign", async () => {
    const root = await mkdtemp(
      path.join(os.tmpdir(), "runner-e2e-report-rerun-test-"),
    );
    cleanupDirectories.push(root);
    const executionId = "runner-opencode.local.ask-question";
    const common: RunnerE2EResult = {
      schema: "paperclip.runner-e2e.result/v1",
      executionId,
      attempt: 2,
      status: "failed",
      failureClass: "candidate_failure",
      profileId: "runner-opencode",
      environmentId: "local",
      caseId: "ask-question",
      provider: "opencode",
      model: "fixture-model",
      runtimeMode: "native",
      startedAt: "2026-08-26T00:00:00.000Z",
      finishedAt: "2026-08-26T00:00:01.000Z",
      durationMs: 1_000,
      cleanup: "not_started",
    };
    const failedDirectory = path.join(root, "old-campaign", "attempt-2");
    const passedDirectory = path.join(root, "new-campaign", "attempt-1");
    await mkdir(failedDirectory, { recursive: true });
    await mkdir(passedDirectory, { recursive: true });
    await writeFile(
      path.join(failedDirectory, "result.json"),
      JSON.stringify(common),
    );
    await writeFile(
      path.join(failedDirectory, "evidence-manifest.json"),
      JSON.stringify({ files: [], leaks: [], missing: [] }),
    );
    await writeFile(
      path.join(passedDirectory, "result.json"),
      JSON.stringify({
        ...common,
        attempt: 1,
        status: "passed",
        failureClass: undefined,
        finishedAt: "2026-08-26T00:00:02.000Z",
        cleanup: "passed",
      }),
    );
    await writeFile(
      path.join(passedDirectory, "evidence-manifest.json"),
      JSON.stringify({
        files: ["final-state.png"],
        leaks: [],
        missing: [],
      }),
    );
    await writeFile(path.join(passedDirectory, "final-state.png"), "fake-png");
    const output = path.join(root, "merged");
    await execFileAsync(
      process.execPath,
      [
        path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"),
        path.join(repositoryRoot, "tests/runner-e2e/report.ts"),
      ],
      {
        cwd: repositoryRoot,
        env: {
          ...process.env,
          PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root,
          PAPERCLIP_RUNNER_E2E_REPORT_OUT: output,
          PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]),
        },
      },
    );
    const normalized = JSON.parse(
      await readFile(path.join(output, "normalized-results.json"), "utf8"),
    );
    expect(normalized).toMatchObject({ passed: 1, failed: 0 });
    expect(normalized.results[0]).toMatchObject({
      attempt: 1,
      status: "passed",
      evidenceValid: true,
    });
  });

  it("constructs the public root JUnit from fixed markup and escaped fields", async () => {
    const root = await mkdtemp(
      path.join(os.tmpdir(), "runner-e2e-report-junit-test-"),
    );
    cleanupDirectories.push(root);
    const executionId = "legacy-codex.local.message-marker";
    const directory = path.join(root, "attempt-1");
    await mkdir(directory, { recursive: true });
    await writeFile(
      path.join(directory, "result.json"),
      JSON.stringify({
        schema: "paperclip.runner-e2e.result/v1",
        executionId,
        attempt: 1,
        status: "failed",
        failureClass: "candidate_failure",
        error: `provider said \"><script>alert(1)</script>&`,
        profileId: "legacy-codex",
        environmentId: "local",
        caseId: "message-marker",
        provider: "codex",
        model: "fixture-model",
        runtimeMode: "legacy",
        startedAt: "2026-08-26T00:00:00.000Z",
        finishedAt: "2026-08-26T00:00:01.000Z",
        durationMs: 1_000,
        cleanup: "passed",
      } satisfies RunnerE2EResult),
    );
    await writeFile(
      path.join(directory, "evidence-manifest.json"),
      JSON.stringify({ files: [], leaks: [], missing: [] }),
    );
    const output = path.join(root, "merged");

    await expect(
      execFileAsync(
        process.execPath,
        [
          path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"),
          path.join(repositoryRoot, "tests/runner-e2e/report.ts"),
        ],
        {
          cwd: repositoryRoot,
          env: {
            ...process.env,
            PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root,
            PAPERCLIP_RUNNER_E2E_REPORT_OUT: output,
            PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]),
          },
        },
      ),
    ).rejects.toBeDefined();

    const junit = await readFile(path.join(output, "junit.xml"), "utf8");
    expect(junit).toContain(
      `message="provider said &quot;&gt;&lt;script&gt;alert(1)&lt;/script&gt;&amp;"`,
    );
    expect(junit).not.toContain("<script>");
    expect(junit).not.toContain("<?xml-stylesheet");
  });
});