import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { runnerMatrix } from "./catalog.js"; import { buildRunnerCampaign, campaignHistoryRecord, emptyRunnerHistory } from "./history.js"; import { fullSuiteSelection, historyBrowserCampaign, historyBrowserJson, historySuiteCatalog, historySuiteCoverage } from "./history-browser.js"; import { refreshHistoryIndex } from "./history-refresh.js"; import type { RunnerE2EResult } from "./types.js"; const temporary: string[] = []; afterEach(async () => { await Promise.all(temporary.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); function fixture() { const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: "app-basics.browser.daytona.create", suiteId: "app-basics", suiteDefinitionHash: "recorded-definition", profileId: "browser", environmentId: "daytona", caseId: "create", provider: null, model: null, runtimeMode: "none", targetUrl: "https://qa.test.paperclip.app/", attempt: 1, status: "passed", cleanup: "passed", startedAt: "2026-10-27T12:00:00Z", finishedAt: "2026-10-27T12:00:01Z", durationMs: 1000 }; const campaign = buildRunnerCampaign({ campaignId: "qa-unchanged", generatedAt: result.finishedAt, expected: [result.executionId], results: [result] }); return { result, campaign, record: campaignHistoryRecord(campaign, "https://reports.example.test/e2e") }; } describe("E2E history organization", () => { it("includes unrun catalog suites and historical suites in one catalog", () => { const history = { ...emptyRunnerHistory(), campaigns: [fixture().record] }; const suites = historySuiteCatalog(history, runnerMatrix); expect(suites.find(suite => suite.id === "app-basics")).toMatchObject({ campaignCount: 1 }); expect(suites.find(suite => suite.id === "core-compatibility")).toMatchObject({ campaignCount: 0 }); expect(new Set(suites.map(suite => suite.id)).size).toBe(suites.length); }); it("uses the recorded full-selection verdict and denominator, independently of failures", () => { const suite = { ...fixture().record.suites[0]!, expected: 8, selected: 8, complete: true, passed: 5, failed: 3 }; expect(fullSuiteSelection(suite)).toBe(true); expect(fullSuiteSelection({ ...suite, selected: 1, passed: 1, failed: 0 })).toBe(false); expect(fullSuiteSelection({ ...suite, complete: false })).toBe(false); expect(fullSuiteSelection({ ...suite, expected: 0, selected: 0 })).toBe(false); expect(historySuiteCoverage({ ...suite, executed: 8 })).toBe("full"); expect(historySuiteCoverage({ ...suite, executed: 0 })).toBe("unfinished"); expect(historySuiteCoverage({ ...suite, selected: 1 })).toBe("subset"); expect(historySuiteCoverage({ ...suite, complete: false })).toBe("unknown"); }); it("records target and agent execution separately without inventing historical targets", () => { const { record } = fixture(); expect(record.executions[0]).toMatchObject({ targetUrl: "https://qa.test.paperclip.app/", runtimeMode: "none", environmentId: "daytona" }); const row = historyBrowserCampaign(record); expect(row.executions[0]).toMatchObject({ target: "https://qa.test.paperclip.app", environment: "daytona", provider: "__none__", model: "__none__" }); delete record.executions[0]!.targetUrl; delete record.executions[0]!.runtimeMode; expect(historyBrowserCampaign(record).executions[0]).toMatchObject({ target: "__unknown__", model: "__unknown__" }); record.executions[0]!.targetUrl = "https://user:secret@example.test"; expect(historyBrowserCampaign(record).executions[0]!.target).toBe("__unknown__"); }); it("keeps public metadata inert inside the embedded JSON script", () => { const { record } = fixture(); record.source.ref = ""; const json = historyBrowserJson({ ...emptyRunnerHistory(), campaigns: [record] }, []); expect(json).not.toContain("<"); expect(JSON.parse(json).campaigns[0].search).toContain(record.source.ref); }); it("imports a reviewed QA measurement as an ordinary campaign and preserves original files and results", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "e2e-history-refresh-")); temporary.push(root); const original = JSON.stringify(emptyRunnerHistory()); const { campaign } = fixture(); const measured = JSON.stringify(campaign); await writeFile(path.join(root, "history.json"), original); await writeFile(path.join(root, "campaign.json"), measured); const output = path.join(root, "output"); const history = await refreshHistoryIndex({ history: path.join(root, "history.json"), output, publicCampaign: path.join(root, "campaign.json"), publicUrl: "https://pages.paperclip.ing/evals-qa-basics/", preset: "qa-smoke" }); expect(history.campaigns[0]).toMatchObject({ campaignId: campaign.campaignId, generatedAt: campaign.generatedAt, source: campaign.source, suites: campaign.suites, selected: campaign.selected, passed: campaign.passed, preset: "qa-smoke" }); expect(await readFile(path.join(root, "history.json"), "utf8")).toBe(original); expect(await readFile(path.join(root, "campaign.json"), "utf8")).toBe(measured); expect(await readFile(path.join(output, "index.html"), "utf8")).toContain('data-index-preset-link="qa-smoke"'); await expect(refreshHistoryIndex({ history: path.join(output, "history.json"), output, publicCampaign: path.join(root, "campaign.json"), publicUrl: "https://pages.paperclip.ing/evals-qa-basics/" })).rejects.toThrow("already exists"); }); it("refuses private Cloud bundles before creating a public history", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "e2e-private-history-")); temporary.push(root); const { campaign } = fixture(); campaign.results[0]!.productJourney = { privacy: "private" } as NonNullable; await writeFile(path.join(root, "history.json"), JSON.stringify(emptyRunnerHistory())); await writeFile(path.join(root, "private.json"), JSON.stringify(campaign)); await expect(refreshHistoryIndex({ history: path.join(root, "history.json"), output: path.join(root, "output"), publicCampaign: path.join(root, "private.json"), publicUrl: "https://pages.paperclip.ing/private/" })).rejects.toThrow("Private Product E2E campaigns"); }); });