diff --git a/doc/evals.md b/doc/evals.md index 5dad409c2b6..f3bd414277a 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -1,5 +1,43 @@ # Paperclip evaluation guide +The public directory calls full-product testing **E2E tests**. Suites describe +what is tested; deployment targets describe which app instance was tested. +QA smoke is a named selection of existing journeys against QA, and appears in +the same campaign history as other E2E runs. Agent execution (local or Daytona), +provider, model and profile are independent filters. Browser-only journeys need +no agent or model. Historical records without target metadata remain explicitly +"Not recorded" rather than being assigned an inferred environment. + +The history sidebar includes the current suite catalog and retained historical +suites. Each suite has its latest matching run and latest full run, with full +coverage determined from the recorded suite definition, selection and execution +counts. An unexecuted full selection remains unfinished, and a subset +can pass without qualifying as a full suite run; failed full selections remain +visible. Changing a model filter does not redefine the suite's recorded coverage. +Existing `runner-e2e` URLs, test IDs, commands, and measurement timestamps remain +stable. **Runner protocol evals** remain a separate category because they test +the runner/provider against a mock control plane. + +Sidebar suite links open that suite's campaign history with other filters cleared. +The Suite dropdown combines with the current filters instead. Explicit URLs retain +their selected filters; when a suite is absent from a preset, the empty state +explains the combination and links to the suite's runs. +Active filters have blue borders and tinted fields. The filter toolbar counts +active values, including a count beside More filters for collapsed controls. +Reset all filters clears every campaign filter and the sidebar suite search. + +`node --import ./cli/node_modules/tsx/dist/loader.mjs tests/runner-e2e/history-browser.browser.mjs` +exercises the history UI using synthetic metadata and Chromium, with no model calls. It checks suite coverage, correlated +suite/target/model/execution filters, saved QA selections, URL history and mobile +navigation. Install the existing Playwright Chromium dependency before running. +For presentation-only refreshes, +`node --import ./cli/node_modules/tsx/dist/loader.mjs tests/runner-e2e/history-refresh.ts --history --output ` +renders an existing history file. Its optional `--public-campaign`, `--public-url` and +`--preset` arguments import a reviewed public measurement without replacing +existing campaigns. Private Cloud bundles are rejected. The output contains +only index HTML and history/pointer JSON; it does not upload evidence or change +retained campaign bundles. + The explicit-only [chat feature evals](../tests/runner-e2e/CHAT-FEATURES.md) promote two natural-language task scenarios into local native Product E2E cells. They verify saving an unstarted task and delegating a saved announcement back to chat, diff --git a/scripts/evals-hub/template.html b/scripts/evals-hub/template.html index 45c94132f4a..b484b281272 100644 --- a/scripts/evals-hub/template.html +++ b/scripts/evals-hub/template.html @@ -3,7 +3,7 @@ - + Paperclip Evals - ${renderReportShell({ history })} + ${renderReportShell({ history, historySuites: suites })}

Historical test reporting

-

Runner E2E campaigns

+

E2E tests

${campaigns.length} retained campaigns. Open a report for test outcomes, per-test billing, declared screenshots, and normalized results.

@@ -242,7 +274,25 @@ export function renderRunnerHistoryIndex( ${latestGreen ? `Latest complete green · ${html(latestGreen.campaignId)}` : ""} -

Recent campaigns

+ +

Filters

None active
+
+ ${filter("suite", "Suite", suites.map(suite => [suite.id, suite.label]))} + ${filter("coverage", "Campaign coverage", [["full", "Full runs"], ["subset", "Subset selections"], ["unfinished", "Unfinished full selections"], ["unknown", "Coverage not established"]])} + ${filter("target", "Target")} + +
+
More filters
+ ${filter("environment", "Agent execution")}${filter("provider", "Provider")}${filter("model", "Model")} + ${filter("profile", "Agent profile")}${filter("runtime", "Runtime")}${filter("preset", "Run preset")} + ${filter("status", "Result", [["passed", "Passed"], ["failed", "Failed"], ["incomplete", "Incomplete"]])} + + +
+

Campaigns

@@ -250,39 +300,12 @@ export function renderRunnerHistoryIndex( ${rows}
- + +
Updated ${html(date(history.updatedAt))} UTCImmutable campaign reports · Declared screenshots and normalized results
- + `; diff --git a/tests/runner-e2e/history-refresh.ts b/tests/runner-e2e/history-refresh.ts new file mode 100644 index 00000000000..425746cad1c --- /dev/null +++ b/tests/runner-e2e/history-refresh.ts @@ -0,0 +1,50 @@ +import path from "node:path"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { parseArgs } from "node:util"; +import { renderRunnerHistoryIndex } from "./history-index.js"; +import { campaignHistoryRecord, mergeRunnerHistory } from "./history.js"; +import { assertPublicCampaign, buildHistoryPointers } from "./history-publish.js"; +import { validateRetainedRunnerResult } from "./result-validation.js"; +import type { RunnerE2ECampaign, RunnerE2EHistoryIndex } from "./types.js"; + +/** Index-only refresh: no provider calls, campaign rewrites, or evidence uploads. */ +export async function refreshHistoryIndex(input: { + history: string; output: string; publicCampaign?: string; publicUrl?: string; preset?: string; +}) { + let history = JSON.parse(await readFile(input.history, "utf8")) as RunnerE2EHistoryIndex; + if (history.schema !== "paperclip.runner-e2e.history/v1" || !Array.isArray(history.campaigns)) throw new Error("Invalid E2E history"); + if (input.publicCampaign) { + const url = new URL(input.publicUrl ?? ""); + if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash) throw new Error("A public HTTPS campaign URL is required"); + if (input.preset && !/^[a-z0-9][a-z0-9-]{0,63}$/.test(input.preset)) throw new Error("Invalid run preset"); + const campaign = JSON.parse(await readFile(input.publicCampaign, "utf8")) as RunnerE2ECampaign; + if (campaign.schema !== "paperclip.runner-e2e.campaign/v2" || !Array.isArray(campaign.results)) throw new Error("A reviewed public campaign is required"); + assertPublicCampaign(campaign); + for (const result of campaign.results) validateRetainedRunnerResult(result); + const record = { ...campaignHistoryRecord(campaign, url.origin), publicUrl: url.href, + ...(input.preset ? { preset: input.preset } : {}) }; + if (history.campaigns.some(existing => existing.campaignId === record.campaignId)) { + throw new Error("Campaign already exists in history; existing measurements are never replaced by an index refresh"); + } + history = mergeRunnerHistory(history, record); + } else if (input.publicUrl || input.preset) throw new Error("Campaign metadata requires a reviewed public campaign"); + const html = renderRunnerHistoryIndex(history); + const pointers = buildHistoryPointers(history); + await mkdir(input.output, { recursive: true }); + await Promise.all([ + writeFile(path.join(input.output, "index.html"), html), + writeFile(path.join(input.output, "history.json"), JSON.stringify(history, null, 2) + "\n"), + writeFile(path.join(input.output, "latest.json"), JSON.stringify(pointers.latest, null, 2) + "\n"), + writeFile(path.join(input.output, "latest-green.json"), JSON.stringify(pointers.latestGreen, null, 2) + "\n"), + ]); + return history; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(import.meta.filename)) { + const { values } = parseArgs({ options: { history: { type: "string" }, output: { type: "string" }, + "public-campaign": { type: "string" }, "public-url": { type: "string" }, preset: { type: "string" } } }); + if (!values.history || !values.output) throw new Error("--history and --output are required"); + const history = await refreshHistoryIndex({ history: values.history, output: values.output, + publicCampaign: values["public-campaign"], publicUrl: values["public-url"], preset: values.preset }); + console.log(`Rendered E2E history with ${history.campaigns.length} retained campaigns; no provider calls.`); +} diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts index b6216af3745..2602b1015ba 100644 --- a/tests/runner-e2e/history.test.ts +++ b/tests/runner-e2e/history.test.ts @@ -179,7 +179,7 @@ describe("runner E2E campaign history", () => { expect(history.latestGreenCampaignId).toBeNull(); expect(history.latestGreenBySuite["first-task"]).toBeUndefined(); expect(renderRunnerHistoryIndex(history)).toContain("1 incomplete"); - expect(renderRunnerHistoryIndex(history)).toContain('data-history-status="incomplete"'); + expect(renderRunnerHistoryIndex(history)).toContain('data-index-status="incomplete"'); }); it("shows unknown campaign costs without plotting zero dollars", () => { @@ -319,8 +319,8 @@ describe("runner E2E campaign history", () => { expect(dashboard).not.toContain('class="history-section"'); const index = renderRunnerHistoryIndex(history); expect(index).toContain("Campaign trends"); - expect(index).toContain("data-history-from"); - expect(index).toContain("data-history-through"); + expect(index).toContain('data-index-filter="from"'); + expect(index).toContain('data-index-filter="through"'); expect(index).toContain( 'data-history-suite-trends="core-compatibility"', ); @@ -333,7 +333,7 @@ describe("runner E2E campaign history", () => { expect(index).toContain("Suite pass rate"); expect(index).toContain("lines break at definition changes"); expect(index).toContain("cleanup passed"); - expect(index).toContain("Runner E2E campaigns"); + expect(index).toContain("E2E tests"); expect(index).toContain("complete-green"); expect(index).toContain("complete-red"); expect(index).toContain(`${runnerMatrix.length}/${runnerMatrix.length} passed`); diff --git a/tests/runner-e2e/history.ts b/tests/runner-e2e/history.ts index 822bd81b7e8..e38c4945c4c 100644 --- a/tests/runner-e2e/history.ts +++ b/tests/runner-e2e/history.ts @@ -1,4 +1,5 @@ import { isIncompleteFirstTaskResult } from "./case-outcome.js"; +import { reportTargetUrl } from "./report-target.js"; import { buildCampaignAccounting } from "./attempt-accounting.js"; import { validateEvalOperationsReport } from "./operations-report.js"; import { validateCampaignFeedbackReview } from "./feedback-dispositions.js"; @@ -269,6 +270,8 @@ export function campaignHistoryRecord( caseId: result.caseId, provider: result.provider, model: result.model, + ...(reportTargetUrl(result) ? { targetUrl: reportTargetUrl(result)! } : {}), + runtimeMode: result.runtimeMode, status: isIncompleteFirstTaskResult(result) ? "incomplete" : result.status, durationMs: result.durationMs, attempt: result.attempt, diff --git a/tests/runner-e2e/report-browser.ts b/tests/runner-e2e/report-browser.ts index d8dd0468e29..35d4c3753c6 100644 --- a/tests/runner-e2e/report-browser.ts +++ b/tests/runner-e2e/report-browser.ts @@ -54,6 +54,7 @@ export const reportBrowserScript = String.raw` } else appendCrumb(view === 'attention' ? 'Needs attention' : view === 'case' ? 'Case unavailable' : 'All results'); }; const apply = () => { + controls.forEach(control => { control.dataset.filterActive = String(Boolean(control.value)); }); const search = query.value.trim().toLowerCase(); const matchesFilters = card => (!search || card.dataset.reportSearch.includes(search)) && diff --git a/tests/runner-e2e/report-catalog.ts b/tests/runner-e2e/report-catalog.ts index 229faa14cea..b0b606df2b6 100644 --- a/tests/runner-e2e/report-catalog.ts +++ b/tests/runner-e2e/report-catalog.ts @@ -2,6 +2,9 @@ import { validateRetainedRunnerResult } from "./result-validation.js"; import { createHash } from "node:crypto"; import type { MatrixExecution, RunnerE2EResult } from "./types.js"; +/** Presentation aliases only; retained execution identities never change. */ +export const retainedSuiteLabels: Readonly> = { "qa-basics": "Basic app journeys" }; + /** Display-only metadata: never load or reconstruct executable branch fixtures. */ export interface ReportExecution { id: string; @@ -92,7 +95,7 @@ export function discoverReportCatalog(input: { .sort(); const suite = knownSuites.get(identity.suiteId) ?? { id: identity.suiteId, - label: identity.suiteId, + label: retainedSuiteLabels[identity.suiteId] ?? identity.suiteId, description: "Suite discovered from retained campaign identities; full suite size is not known to this publisher.", expectedMatrixSize: suiteIds.length, diff --git a/tests/runner-e2e/report-history.ts b/tests/runner-e2e/report-history.ts index d83055a4250..a31bdf160c0 100644 --- a/tests/runner-e2e/report-history.ts +++ b/tests/runner-e2e/report-history.ts @@ -67,7 +67,7 @@ function suiteSummaryFor( return campaign.suites.find((suite) => suite.suiteId === suiteId); } -export function renderReportHistory(history: RunnerE2EHistoryIndex | undefined) { +export function renderReportHistory(history: RunnerE2EHistoryIndex | undefined, options: { chartsOnly?: boolean } = {}) { if (!history || history.campaigns.length === 0) { return `

History

Campaign trends

No historical campaigns have been published yet.

`; } @@ -211,7 +211,7 @@ export function renderReportHistory(history: RunnerE2EHistoryIndex | undefined)
${charts}
${suiteCharts} -
+ ${options.chartsOnly ? "" : `
@@ -220,7 +220,7 @@ export function renderReportHistory(history: RunnerE2EHistoryIndex | undefined)
${rows}
CampaignPaperclip SHAResultTokensCostExecution
- + `} `; } diff --git a/tests/runner-e2e/report-navigation.test.ts b/tests/runner-e2e/report-navigation.test.ts index f978550cb4e..1ef8a42d1ac 100644 --- a/tests/runner-e2e/report-navigation.test.ts +++ b/tests/runner-e2e/report-navigation.test.ts @@ -14,7 +14,8 @@ describe("evaluation report navigation", () => { expect(sidebar).toContain("Overview"); expect(sidebar).toContain("Needs attention"); expect(sidebar).toContain("Campaign history"); - expect(sidebar).not.toContain("Suites"); + expect(sidebar).toContain('aria-label="E2E suites"'); + expect(sidebar).toContain('data-index-suite-link="core-compatibility"'); expect(page).not.toContain('aria-label="Campaign suites"'); expect(page).not.toContain('data-nav-campaign aria-label="Campaign"'); }); @@ -42,9 +43,9 @@ describe("evaluation report navigation", () => { const history = emptyRunnerHistory(); history.campaigns = [{campaignId:"campaign-a",publicUrl:"/custom/reports/campaigns/campaign-a/"} as typeof history.campaigns[number]]; expect(reportHistoryHref(history,"campaign-a")).toBe("/custom/reports/"); - expect(reportHistoryHref(undefined,"campaign-a")).toBe("https://pages.paperclip.ing/evals/"); + expect(reportHistoryHref(undefined,"campaign-a")).toBe("https://d1p6rlowie26tp.cloudfront.net/runner-e2e/"); history.campaigns[0]!.publicUrl = "https://pages.paperclip.ing/standalone-report/"; - expect(reportHistoryHref(history,"campaign-a")).toBe("https://pages.paperclip.ing/evals/"); + expect(reportHistoryHref(history,"campaign-a")).toBe("https://d1p6rlowie26tp.cloudfront.net/runner-e2e/"); expect(page).toContain('class="brand-lockup report-brand" href="https://pages.paperclip.ing/evals/"'); }); }); diff --git a/tests/runner-e2e/report-navigation.ts b/tests/runner-e2e/report-navigation.ts index 58a86f40c17..ea1cf5daa0e 100644 --- a/tests/runner-e2e/report-navigation.ts +++ b/tests/runner-e2e/report-navigation.ts @@ -12,13 +12,14 @@ export function reportHistoryHref(history: RunnerE2EHistoryIndex | undefined, ca const url = history?.campaigns.find(campaign => campaign.campaignId === campaignId)?.publicUrl; return url && /campaigns\/[^/]+\/(?:index\.html)?$/.test(url) ? url.replace(/campaigns\/[^/]+\/(?:index\.html)?$/, "") - : "https://pages.paperclip.ing/evals/"; + : "https://d1p6rlowie26tp.cloudfront.net/runner-e2e/"; } export function renderReportShell(input: { campaignId?: string; catalog?: readonly ReportExecution[]; history?: RunnerE2EHistoryIndex; + historySuites?: readonly { id: string; label: string; campaignCount: number }[]; }) { const h = reportHtml; const campaign = Boolean(input.catalog); @@ -28,7 +29,8 @@ export function renderReportShell(input: { const tree = campaign ? `` : ""; + }).join("")}` : input.historySuites ? `` : ""; + const presets = [...new Set(input.history?.campaigns.map(row => row.preset).filter((preset): preset is string => Boolean(preset)) ?? [])]; return `
Product E2E${campaign ? `${input.catalog!.length} executions` : ""}
+ ${!campaign && presets.length ? `` : ""}${tree}
E2E tests${campaign ? `${input.catalog!.length} executions` : ""}
@@ -61,6 +63,10 @@ export const reportShellScript = ` export const reportNavigationStyles = ` :root { --report-sidebar-width: 250px; --navigation-sticky-offset: 64px; } + :root { --report-filter-active:#2563eb; --report-filter-tint:#eff6ff; } + @media(prefers-color-scheme:dark) { :root { --report-filter-active:#60a5fa; --report-filter-tint:#172235; } } + .report-shell [data-filter-active=true] { border-color:var(--report-filter-active); background-color:var(--report-filter-tint); box-shadow:0 0 0 1px var(--report-filter-active); } + .report-shell label:has([data-filter-active=true]) > span { color:var(--report-filter-active); } html { scroll-behavior: auto; } [hidden] { display: none !important; } .report-shell { min-height: 100dvh; } @@ -76,6 +82,12 @@ export const reportNavigationStyles = ` .report-global-nav a:hover,.report-suite-tree a:hover { background: var(--raised); color: var(--foreground); } .report-global-nav a[aria-current=page] { color: var(--foreground); background: var(--raised); } .report-side-label { padding: 18px 10px 7px; margin: 0; font: 10px var(--font-mono); color: var(--muted-foreground); text-transform: uppercase; letter-spacing: .08em; } + .report-suite-catalog { display: grid; gap: 2px; } + .report-suite-catalog input { min-width: 0; width: calc(100% - 12px); margin: 0 6px 7px; padding: 8px; border: 1px solid var(--border); border-radius: 5px; color: var(--foreground); background: var(--background); font: 12px var(--font-sans); } + .report-suite-catalog a { display: flex; align-items: baseline; justify-content: space-between; gap: 10px; padding: 8px 10px; color: var(--muted-foreground); text-decoration: none; border-radius: 5px; font-size: 12px; } + .report-suite-catalog a span { min-width: 0; overflow-wrap: anywhere; } + .report-suite-catalog a small { flex: none; font: 10px var(--font-mono); } + .report-suite-catalog a:hover,.report-suite-catalog a[aria-current=page] { color: var(--foreground); background: var(--raised); } .report-suite-tree details > summary { display: flex; align-items: center; gap: 4px; list-style: none; cursor: pointer; padding-left: 4px; } .report-suite-tree summary::-webkit-details-marker { display: none; } .report-suite-tree summary::before { content: '›'; width: 12px; flex: none; color: var(--muted-foreground); } @@ -94,6 +106,15 @@ export const reportNavigationStyles = ` .report-breadcrumbs [aria-current=page] { color: var(--foreground); } .report-breadcrumbs a,.report-breadcrumbs span { overflow-wrap: anywhere; } .report-campaign-select { font: 11px var(--font-sans); padding: 8px; color: var(--foreground); background: var(--card); border: 1px solid var(--border); border-radius: 5px; width: 220px; max-width: 30%; flex: none; } + .report-shell .report-surface select { + appearance: none; + -webkit-appearance: none; + padding-inline-end: 34px; + background-image: linear-gradient(45deg, transparent 50%, var(--muted-foreground) 50%), linear-gradient(135deg, var(--muted-foreground) 50%, transparent 50%); + background-position: calc(100% - 16px) center, calc(100% - 12px) center; + background-size: 4px 4px; + background-repeat: no-repeat; + } .report-menu-toggle { display: none; border: 0; padding: 0; background: transparent; color: var(--foreground); font-size: 20px; cursor: pointer; } .report-drawer-backdrop { display: none; } .report-header h1 { font-size: 32px; } diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index 259c0d756c7..950750f0984 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -398,7 +398,7 @@ describe("runner E2E report aggregation", () => { path.join(output, "dashboard.html"), "utf8", ); - expect(dashboard).toContain("Runner Full-Stack E2E"); + expect(dashboard).toContain("E2E tests"); expect(dashboard).toMatch(/\$0\.0250*<\/strong>LLM reported subtotal/); expect(dashboard).toContain("2sSummed attempt time"); expect(dashboard).toContain("Carried-forward evidence can span older runs"); diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts index a8a9664c3ac..7167e7316c1 100644 --- a/tests/runner-e2e/report.ts +++ b/tests/runner-e2e/report.ts @@ -363,7 +363,7 @@ async function main() { `${JSON.stringify(normalized, null, 2)}\n`, ); const dashboard = renderRunnerE2EDashboard({ - title: "Runner Full-Stack E2E", + title: "E2E tests", generatedAt: normalized.generatedAt, expected, catalog: runnerMatrix, @@ -393,7 +393,7 @@ async function main() { )?.url; const summaryLines = [ - "# Runner Full-Stack E2E", + "# E2E tests", "", `Passed: ${normalized.passed}/${selected.length}`, "", diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 0f2fe4c9b94..e5c672538c1 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -3342,6 +3342,7 @@ for (const execution of executions) { ...(completionQuality.length ? { completionQuality } : {}), ...(publicMcpUsage ? { publicMcp: publicMcpUsage } : {}), runtimeMode: execution.profile.expectedRuntimeMode, + targetUrl: testInfo.project.use.baseURL, issueId: issue?.id, issueIdentifier: issue?.identifier ?? null, runIds: selectedRuns.map((run) => run.id), diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index 04eec391440..0feaf41a0c3 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -477,6 +477,9 @@ export interface RunnerE2EHistoryExecution { caseId: string; provider: string | null; model: string | null; + /** Absent in older records; never infer the application from the worker. */ + targetUrl?: string; + runtimeMode?: RunnerE2EResult["runtimeMode"]; status: "passed" | "failed" | "incomplete"; durationMs: number; attempt: number; @@ -500,6 +503,8 @@ export interface RunnerE2EHistoryCampaign { billing: RunnerE2EAggregateBillingSummary; suites: RunnerE2ESuiteSummary[]; executions: RunnerE2EHistoryExecution[]; + /** A named case selection, independent of suite and deployment target. */ + preset?: string; } export interface RunnerE2EHistoryIndex {