From 8279c990080b15abdcd9981f5793290fab37f9ad Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:00:54 -0500 Subject: [PATCH] fix(evals): clarify report details and link tested applications (#18009) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Product E2E reports help operators inspect real application journeys. > - A worker environment does not identify the application that a journey tested. > - Reports hid that URL and gave missing diagnostics too much space. > - Standalone reports also sent the logo link to a missing parent page. > - This pull request shows the recorded target and checks in the shared report. > - The benefit is a clear report with useful links and inspectable diagnostics. ## Linked Issues or Issue Description **What happened?** A standalone Product E2E report had no application link in test details or the screenshot gallery. Its details showed several empty diagnostic sections before useful information. The logo could resolve to the page host root and return 404. Campaign notes added outside the report shell could overlap the fixed sidebar. **Expected behavior** Show the recorded application URL beside each test and screenshot. Put recorded checks before optional diagnostics. Keep campaign context inside the shared layout. Link the logo to the evaluations directory. **Steps to reproduce** 1. Render a retained campaign at a standalone page path without a history index. 2. Open a test with screenshot evidence and missing startup timing. 3. Inspect test details and gallery information, then click the report logo. **Paperclip version or commit** Reproduced in the report renderer at `39e811c9c5`. This branch applies the fix to current master. **Deployment mode** Static Product E2E HTML report generated from retained normalized results. ## What Changed - Add optional recorded `targetUrl` metadata and resolve existing Product E2E and provider-connection targets. Only HTTP and HTTPS links without embedded credentials are shown. - Show the target URL in test details and screenshot gallery information. - Put recorded checks and execution facts first. Keep raw usage, timing, and startup details under Diagnostics. Preserve missing metadata as unknown. - Keep campaign titles, descriptions, notes, and evidence links inside the shared report layout during retained-evidence regeneration. - Fix standalone navigation and let the brand lockup wrap at narrow sidebar widths. - Match report profiles by suite and recorded provider/model before falling back to an inferred profile. - Add report regressions and document the retained metadata contract. ## Verification - Focused report tests: 86 passed across 8 files on the isolated branch. Command: `node node_modules/vitest/vitest.mjs run --config tests/runner-e2e/vitest.config.ts report-target.test.ts report-navigation.test.ts report-catalog.test.ts first-task-report.test.ts report.test.ts dashboard-regenerate.test.ts`. - Browser verification covered dark desktop widths 1440 and 1000, light intermediate width 780, and dark mobile width 390. It covered the drawer, URL links, gallery keyboard navigation, focus return, and the logo destination. - The published retained report keeps all 3 original passing cases and 7 recorded checks. Its campaign and screenshot bytes remain unchanged. This presentation refresh made zero provider calls. - Full `pnpm build`, `pnpm -r typecheck`, and the eval TypeScript check passed after rebuilding the reused package dependencies. The first typecheck attempt had stale local dependency builds; those errors are resolved. - The local `pnpm test:run` attempt hit repeated embedded PostgreSQL bootstrap failures in server suites. I stopped that attempt and retained its log. It is not counted as a passing check. Current-head CI remains the full test gate. - Review follow-ups add provider/model and cross-suite profile regressions. Malformed campaign presentation metadata is rejected before any report or retained-result writes; ten malformed input cases verify that retained results stay unchanged. ## Risks - This changes presentation across Product E2E reports. Historical reports without a target show “Not recorded” rather than an inferred URL. - Unknown startup reuse data remains in raw metadata but no longer creates an empty table. - Standalone logo links open the shared public evaluations directory. Campaign history links still use a matching history index when present. - No application runtime, database, or measurement verdict changes are included. Private QA configuration and raw campaign evidence stay outside this PR. ## Model Used - OpenAI Codex, GPT-6 family, with reasoning, code execution, and browser tools. The exact backend model ID and context-window size are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/evals.md | 8 +++ tests/runner-e2e/dashboard-regenerate.ts | 26 +++++++- tests/runner-e2e/dashboard.ts | 70 ++++++++++++++++----- tests/runner-e2e/first-task-report.test.ts | 2 +- tests/runner-e2e/report-catalog.test.ts | 73 +++++++++++++++++++++- tests/runner-e2e/report-catalog.ts | 5 +- tests/runner-e2e/report-navigation.test.ts | 5 +- tests/runner-e2e/report-navigation.ts | 8 ++- tests/runner-e2e/report-target.test.ts | 27 ++++++++ tests/runner-e2e/report-target.ts | 23 +++++++ tests/runner-e2e/report.test.ts | 2 +- tests/runner-e2e/result-validation.ts | 1 + tests/runner-e2e/types.ts | 2 + 13 files changed, 228 insertions(+), 24 deletions(-) create mode 100644 tests/runner-e2e/report-target.test.ts create mode 100644 tests/runner-e2e/report-target.ts diff --git a/doc/evals.md b/doc/evals.md index 6cb2ce4b3da..5dad409c2b6 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -331,6 +331,14 @@ compatible complete campaigns. Refreshing an existing report from retained evidence has zero provider calls and is a new presentation of the old measurement, not a new model run. +Product E2E test details and screenshot galleries show the recorded application +URL when it is available. The worker environment, such as Daytona, does not +imply an application URL. Public projections can retain the reviewed URL in +`targetUrl`. Test details put recorded checks first and keep raw usage, timing, +and startup evidence under Diagnostics. Optional `presentation` metadata in a +retained campaign supplies its title, description, notes, and evidence links +inside the shared report shell when the dashboard is regenerated. + Existing public histories are available at [Runner protocol history](https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/index.html) and [Runner Product E2E history](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/). diff --git a/tests/runner-e2e/dashboard-regenerate.ts b/tests/runner-e2e/dashboard-regenerate.ts index 2913bd96459..98bba7f8d45 100644 --- a/tests/runner-e2e/dashboard-regenerate.ts +++ b/tests/runner-e2e/dashboard-regenerate.ts @@ -1,7 +1,7 @@ import path from "node:path"; import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; import { runnerMatrix } from "./catalog.js"; -import { renderRunnerE2EDashboard } from "./dashboard.js"; +import { renderRunnerE2EDashboard, type RunnerDashboardInput } from "./dashboard.js"; import { campaignFeedbackReviewText } from "./feedback-dispositions.js"; import { buildRunnerCampaign, @@ -30,6 +30,25 @@ interface PublishedCampaign { operations?: RunnerE2ECampaign["operations"]; feedbackReview?: RunnerE2ECampaign["feedbackReview"]; braintrustExperiment?: RunnerE2ECampaign["braintrustExperiment"]; + presentation?: RunnerDashboardInput["presentation"]; +} + +function validatePresentation(value: unknown) { + if (value === undefined) return; + const invalid = () => { + throw new Error("Published bundle has invalid presentation metadata in normalized-results.json"); + }; + if (!value || typeof value !== "object" || Array.isArray(value)) invalid(); + const presentation = value as Record; + for (const field of ["title", "description"]) { + if (presentation[field] !== undefined && typeof presentation[field] !== "string") invalid(); + } + if (presentation.notes !== undefined && + (!Array.isArray(presentation.notes) || presentation.notes.some(note => typeof note !== "string"))) invalid(); + if (presentation.links !== undefined && + (!Array.isArray(presentation.links) || presentation.links.some(link => + !link || typeof link !== "object" || Array.isArray(link) || + typeof link.label !== "string" || typeof link.href !== "string"))) invalid(); } async function relativeFiles(root: string, current = root): Promise { @@ -89,6 +108,7 @@ export async function regenerateRunnerDashboard(input: { ) { throw new Error("Published bundle has an invalid normalized-results.json"); } + validatePresentation(normalized.presentation); const results = normalized.results.map(upgradeRunnerResult); const expected = normalized.expected.map(canonicalExecutionId); @@ -162,16 +182,18 @@ export async function regenerateRunnerDashboard(input: { }), ); const dashboard = renderRunnerE2EDashboard({ - title: "Runner Full-Stack E2E", + title: normalized.presentation?.title ?? "Runner Full-Stack E2E", generatedAt: normalized.generatedAt, expected, catalog: runnerMatrix, entries, campaign, history, + presentation: normalized.presentation, }); const upgraded = { ...campaign, + ...(normalized.presentation ? { presentation: normalized.presentation } : {}), results: campaign.results.map((result, index) => ({ ...result, evidenceValid: diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index b8cb4ac92b0..e8ff48e7705 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -2,12 +2,13 @@ import { campaignAttemptTimeLabel, campaignTimingScope, renderWorkerPhaseTiming, import { html, durationLabel, tokenLabel, llmCostLabel, usdLabel } from "./report-format.js"; import { renderReportShell, reportNavigationStyles, reportShellScript } from "./report-navigation.js"; import { reportBrowserScript } from "./report-browser.js"; +import { reportEvidenceHref, reportTargetUrl } from "./report-target.js"; import { isIncompleteFirstTaskResult, renderCaseOutcome } from "./case-outcome.js"; import { latencyIntervalLabel } from "./latency-timing.js"; import { campaignSuiteSummaryStyles, renderCampaignSuiteSummary, renderSuiteCaseIndex } from "./campaign-summary.js"; import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js"; import { journeyFeatureStyles, renderJourneyFeatureCoverage, renderJourneyFeatureOptions } from "./feature-coverage.js"; -import { STARTUP_REUSE_LAYERS, summarizeStartupReuse } from "../../packages/adapter-utils/src/startup-reuse.js"; +import { STARTUP_REUSE_LAYERS } from "../../packages/adapter-utils/src/startup-reuse.js"; import { renderFirstTaskDetails } from "./first-task-report.js"; import { summarizeCampaignFeedback } from "./feedback.js"; import { renderCampaignFeedbackReview } from "./feedback-dispositions.js"; @@ -45,6 +46,12 @@ export interface RunnerDashboardInput { entries: readonly RunnerDashboardEntry[]; campaign?: RunnerE2ECampaign; history?: RunnerE2EHistoryIndex; + presentation?: { + title?: string; + description?: string; + notes?: readonly string[]; + links?: readonly { label: string; href: string }[]; + }; } interface ResolvedScreenshot { @@ -147,6 +154,7 @@ function renderCase( : item, ); const billing = entry ? summarizeExecutionBilling(entry.result) : null; + const targetUrl = reportTargetUrl(entry?.result); const capabilities = journeyCapabilities(execution); const features = [...new Map(capabilities.map((capability) => [capability.feature.id, capability.feature])).values()]; const outcome = entry ? renderCaseOutcome(entry.result, entry.valid, entry.errors) : ""; @@ -233,9 +241,9 @@ function renderCase( ${html(latencyIntervalLabel(probe))}${html(`${probe.target.operator} ${probe.target.ms} ms`)} ${html(probe.resolutionMs === null ? "unknown" : `${probe.resolutionMs} ms`)} `).join(""); - const startupReuseRows = (entry?.result.startupReuse ?? (entry?.result.runIds ?? []).map(runId => ({ - runId, ...summarizeStartupReuse([]), - }))).map(reuse => ` + const startupReuseRows = (entry?.result.startupReuse ?? []).filter(reuse => + reuse.providerLaunchCount !== null || STARTUP_REUSE_LAYERS.some(layer => reuse.layers[layer].state !== "unknown"), + ).map(reuse => ` ${html(reuse.runId)}${html(reuse.warmth)} ${STARTUP_REUSE_LAYERS.map(layer => `${html(reuse.layers[layer].state)} (${html(reuse.layers[layer].reason)})`).join("")} ${html(reuse.providerLaunchCount === null ? "unknown" : reuse.providerLaunchCount)} @@ -250,6 +258,7 @@ function renderCase( data-gallery-href="${html(item.href)}" data-gallery-label="${html(item.label)}" data-gallery-execution="${html(execution.id)}" + data-gallery-target-url="${html(targetUrl)}" data-gallery-case="${html(execution.task.label)}" data-gallery-profile="${html(execution.profile.label)}" data-gallery-generation="${html(execution.profile.generation)}" @@ -262,7 +271,7 @@ function renderCase( data-gallery-status="${html(label)}" data-gallery-duration="${html(entry ? durationLabel(entry.result.durationMs) : "Not run")}" data-gallery-tokens="${html(billing ? `${tokenLabel(billing.llm.inputTokens, billing.llm)} in · ${tokenLabel(billing.llm.outputTokens, billing.llm)} out` : "Unavailable")}" - data-gallery-matchers="${html(matcherResults.length > 0 ? `${passedMatchers}/${matcherResults.length} matchers passed${notReached ? ` · ${notReached} not reached` : ""}` : "No matchers recorded")}" + data-gallery-matchers="${html(matcherResults.length > 0 ? `${passedMatchers}/${matcherResults.length} checks passed${notReached ? ` · ${notReached} not reached` : ""}` : "No checks recorded")}" aria-label="Open ${html(item.label)} for ${html(execution.id)} in gallery" > ${html(item.label)} for ${html(execution.id)} @@ -276,7 +285,7 @@ function renderCase(
Tokens${html(tokenLabel(billing.llm.inputTokens, billing.llm))} in · ${html(tokenLabel(billing.llm.outputTokens, billing.llm))} out${html(tokenLabel(billing.llm.cachedInputTokens, billing.llm))} cached · ${billing.llm.runsWithTokenUsage}/${billing.llm.runCount} runs covered
LLM spend${html(llmCostLabel(billing.reportedCostUsd, billing.llm))}${billing.llm.runsWithReportedCost}/${billing.llm.runCount} runs provider-priced
${(billing.llm.runsWithEstimatedCost ?? 0) > 0 ? `
LLM estimate${html(billingCoverageLabel(usdLabel(billing.llm.estimatedCostUsd ?? 0), billing.llm.runsWithEstimatedCost ?? 0, billing.llm.runCount))}Estimated from captured usage
` : ""} -
Execution${billing.runtime.estimatedListCostUsd === undefined ? html(billing.runtime.costStatus === "not_metered" ? "Local · not metered" : "Cost unavailable") : `${html(usdLabel(billing.runtime.estimatedListCostUsd))} est.`}${html(durationLabel(billing.runtime.agentRunDurationMs))} agent${billing.runtime.leaseDurationMs === null ? "" : ` · ${html(durationLabel(billing.runtime.leaseDurationMs))} lease`}
+
Execution${billing.runtime.estimatedListCostUsd === undefined ? html(billing.runtime.costStatus === "not_metered" ? "Local · not metered" : "Unavailable") : `${html(usdLabel(billing.runtime.estimatedListCostUsd))} est.`}${billing.runtime.agentRunDurationMs === null && billing.runtime.leaseDurationMs === null ? "" : `${billing.runtime.agentRunDurationMs === null ? "" : `${html(durationLabel(billing.runtime.agentRunDurationMs))} agent`}${billing.runtime.leaseDurationMs === null ? "" : ` · ${html(durationLabel(billing.runtime.leaseDurationMs))} lease`}`}
${billing.assistant ? `
External assistant${html(usdLabel(billing.assistant.estimatedCostUsd))} est.${html(billing.assistant.model)} · ${billing.assistant.requests} requests · ${html(tokenLabel(billing.assistant.inputTokens + billing.assistant.cachedInputTokens))} in / ${html(tokenLabel(billing.assistant.outputTokens))} out
` : ""} ` : ""; @@ -306,13 +315,22 @@ function renderCase( ${gallery}
Test details${matcherResults.length ? `${passedMatchers}/${matcherResults.length} checks${notReached ? ` · ${notReached} not reached` : ""}` : "No checks recorded"}${entry ? ` · ${html(durationLabel(entry.result.durationMs))}` : ""} - ${passed ? outcome : ""} + ${entry ? `
+
Tested URL
${targetUrl ? `${html(targetUrl)} ` : "Not recorded"}
+
Environment
${html(execution.environment.label)}
+ ${entry.result.model ? `
Model
${html(entry.result.model)}
` : ""} +
Runtime
${html(entry.result.runtimeMode === "none" ? "Browser only" : `${entry.result.runtimeMode} · ${entry.result.provider ?? execution.profile.provider}`)}
+
` : ""} + ${matcherResults.length || notReached ? `
    ${matcherResults.map(result => `
  • ${result.passed ? "Pass" : "Fail"}${html(result.detail)}
    • `).join("")}${firstTaskChecks?.filter(check => check.notReached).map(check => `
  • Not reached${html(check.detail)} · ${html(check.notReached)}
    • `).join("") ?? ""}
` : ""} ${billingStrip} + ${renderFirstTaskDetails(entry?.result)} + ${links} +
+ Diagnostics + ${passed ? outcome : ""} ${entry ? `

Worker phase timing: ${entry.result.workerTiming ? html(entry.result.workerTiming.status === "unavailable" ? `unavailable (${entry.result.workerTiming.reason})` : entry.result.workerTiming.status) : "not captured"}. ${entry.result.workerTiming?.status === "partial" ? "Only completed phases have observed durations." : ""}

` : ""} ${entry?.result.feedback ? `

Feedback: ${entry.result.feedback.complaints} complaints · ${entry.result.feedback.suggestions} suggestions · ${entry.result.feedback.review.length} distinct items; see campaign feedback review. Saved-feedback collection: ${html(entry.result.feedback.collection)}; invocation coverage: ${html(entry.result.feedback.invocationCollection)}. Tool receipts: ${entry.result.feedback.invocations.failed} failed · ${entry.result.feedback.invocations.unknown} unresolved. Full bodies: access-controlled snapshots/agent-feedback.json or snapshots/agent-feedback-recovered.json.

` : entry ? `

Feedback was not captured for this attempt.

` : ""} ${entry?.result.tracing ? `

Braintrust worker export: ${html(entry.result.tracing.status)}; native configuration: ${html(entry.result.tracing.nativeConfiguration)}. ${entry.result.tracing.finalization ? `Supervisor outcome: ${html(entry.result.tracing.finalization.outcome)}; finalization export: ${html(entry.result.tracing.finalization.status)}.` : "Supervisor finalization was not captured."} Trace ${html(entry.result.tracing.traceId)} in project ${html(entry.result.tracing.projectId)}. Indexing and native-span correlation require independent verification.

` : ""} - ${links} - ${renderFirstTaskDetails(entry?.result)} ${html(execution.id)} ${capabilities.length ? `
    ${capabilities.map((capability) => `
  • ${html(capability.feature.title)} · ${html(capability.label)}
    • `).join("")}
` : ""} ${ @@ -329,12 +347,13 @@ function renderCase( ` : "" } -

${html(detail)}

+ ${entry?.errors.length ? `

${html(detail)}

` : ""} ${turnTimingRows ? `
${turnTimingRows}
TurnRunLeaseSchedulerRun durationFinal settlement (legacy response field)
` : ""} ${latencyRows ? `
${latencyRows}
EventTiming verdictPossible latencyTargetResolution
` : ""} ${startupReuseRows ? `
${startupReuseRows}
Startup reuse evidence
RunWarmthEnvironment/leaseWorkspaceRunnerProvider processProvider sessionProvider launches
` : ""} - ${matcherRows ? `
${matcherRows}
ResultMatcherExpectationDetail
` : `

No matcher result was recorded.

`} - ${entry ? `
Usage, billing and timing metadata
${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null, workerTiming: entry.result.workerTiming ?? null }, null, 2))}
` : ""} + ${matcherRows ? `
${matcherRows}
ResultMatcherExpectationDetail
` : ""} + ${entry ? `
Usage, billing and timing metadata
${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null, workerTiming: entry.result.workerTiming ?? null, startupReuse: entry.result.startupReuse ?? null }, null, 2))}
` : ""} +
`; } @@ -379,6 +398,10 @@ function renderSuiteResults(input: { } export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { + const presentationLinks = (input.presentation?.links ?? []).flatMap(link => { + const href = reportEvidenceHref(link.href); + return href ? [{ ...link, href }] : []; + }); const expected = new Set(input.expected); const catalog = discoverReportCatalog({ catalog: input.catalog, @@ -662,6 +685,17 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .case-diagnostics > summary::before { content: "›"; color: var(--muted-foreground); font-size: 16px; line-height: 1; } .case-diagnostics[open] > summary::before { transform: rotate(90deg); } .case-detail-summary { margin-left: auto; color: var(--muted-foreground); font: 400 11px/1.5 var(--font-mono); } + .case-facts { display: grid; gap: 10px; margin: 18px 0; font-size: 12px; } + .case-facts > div { display: grid; grid-template-columns: 92px minmax(0, 1fr); align-items: baseline; gap: 12px; } + .case-facts dd { min-width: 0; overflow-wrap: anywhere; font: inherit; } + .case-facts a { text-underline-offset: 3px; } + .case-checks { display: grid; gap: 9px; list-style: none; padding: 14px 0; margin: 0; border-top: 1px solid var(--border); font-size: 12px; } + .case-checks li { display: flex; align-items: baseline; gap: 10px; overflow-wrap: anywhere; } + .check-state { flex: none; min-width: 32px; color: var(--muted-foreground); font-size: 10px; font-weight: 600; } + .check-passed { color: var(--pass-text); } + .check-failed { color: var(--fail-text); } + .case-advanced { margin-top: 18px; } + .case-advanced > summary { color: var(--muted-foreground); font-weight: 400; } .case-capabilities { padding-left: 16px; color: var(--muted-foreground); font-size: 11px; } .case-outcome { margin: 12px 0; padding: 12px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--raised); font-size: 12px; } .case-incomplete > .case-outcome { border-color: var(--missing-border); } @@ -735,6 +769,8 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .gallery-context span { display: inline-flex; align-items: baseline; gap: 6px; min-width: 0; } .gallery-context strong { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-size: 12px; } .gallery-context em { color: #b8b8b5; font: 10px/1.4 var(--font-mono); font-style: normal; white-space: nowrap; } + .gallery-context .gallery-target { flex-basis: 100%; } + .gallery-target a { color: #fafafa; font-size: 12px; overflow-wrap: anywhere; text-underline-offset: 3px; } .gallery-facts { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 18px; margin-top: 8px; } .gallery-facts > span { display: inline-flex; align-items: baseline; gap: 6px; min-width: 0; } .gallery-facts em { color: #b8b8b5; font: 10px/1.4 var(--font-mono); font-style: normal; text-transform: uppercase; } @@ -833,8 +869,8 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {

Full-stack acceptance campaign

-

${html(input.title)}

-

Results across runner profiles and execution environments. Declared PNG screenshots and normalized results are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.

+

${html(input.presentation?.title ?? input.title)}

+

${html(input.presentation?.description ?? "Results across runner profiles and execution environments. Declared PNG screenshots and normalized results are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.")}

${input.campaign ? `

${html(input.generatedAt)}${input.campaign.source.sha ? `${html(input.campaign.source.sha.slice(0,8))}` : ""}${input.campaign.source.workflowRunUrl ? `Source run ↗` : ""}

` : ""}
@@ -847,6 +883,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
+ ${input.presentation?.notes?.length || presentationLinks.length ? `
About this run${input.presentation?.notes?.map(note => `

${html(note)}

`).join("") ?? ""}${presentationLinks.length ? `` : ""}
` : ""} ${renderCampaignSuiteSummary(input.campaign?.suites ?? [], catalog, "", false)}
Campaign usage and spend
@@ -894,6 +931,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { Environment: + Tested URL:
@@ -953,6 +991,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { const profileDetail = dialog.querySelector("[data-gallery-profile-detail]"); const environment = dialog.querySelector("[data-gallery-environment]"); const environmentDetail = dialog.querySelector("[data-gallery-environment-detail]"); + const targetUrl = dialog.querySelector("[data-gallery-target-url]"); const caseLabel = dialog.querySelector("[data-gallery-case]"); const runtime = dialog.querySelector("[data-gallery-runtime]"); const status = dialog.querySelector("[data-gallery-status]"); @@ -983,6 +1022,9 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { profileDetail.textContent = item.dataset.galleryGeneration + " · " + item.dataset.galleryProvider + " · " + item.dataset.galleryModel; environment.textContent = item.dataset.galleryEnvironment; environmentDetail.textContent = "Provider: " + item.dataset.galleryEnvironmentProvider + " · Target: " + item.dataset.galleryExecutionTarget; + targetUrl.textContent = item.dataset.galleryTargetUrl || "Not recorded"; + if (item.dataset.galleryTargetUrl) targetUrl.href = item.dataset.galleryTargetUrl; + else targetUrl.removeAttribute("href"); caseLabel.textContent = item.dataset.galleryCase; runtime.textContent = item.dataset.galleryRuntime + " runtime"; status.textContent = item.dataset.galleryStatus; diff --git a/tests/runner-e2e/first-task-report.test.ts b/tests/runner-e2e/first-task-report.test.ts index 1150d002338..974f1ca8b6d 100644 --- a/tests/runner-e2e/first-task-report.test.ts +++ b/tests/runner-e2e/first-task-report.test.ts @@ -265,7 +265,7 @@ describe("first-task conversation report", () => { }); const page = render(); expect(page).toContain('data-gallery-status="incomplete"'); - expect(page).toContain('data-gallery-matchers="1/1 matchers passed · 1 not reached"'); + expect(page).toContain('data-gallery-matchers="1/1 checks passed · 1 not reached"'); expect(page).toContain('data-gallery-label="Task state when recording stopped"'); expect(page).toContain("Incomplete journey"); expect(page).toContain('1Incomplete'); diff --git a/tests/runner-e2e/report-catalog.test.ts b/tests/runner-e2e/report-catalog.test.ts index 5cd96f1e078..0cb16af378c 100644 --- a/tests/runner-e2e/report-catalog.test.ts +++ b/tests/runner-e2e/report-catalog.test.ts @@ -187,6 +187,68 @@ describe("trusted report catalog discovery", () => { ).toThrow("must be unique"); }); + it("keeps the suite's profile when another suite uses the same profile ID", () => { + const known = runnerMatrix[0]!; + const other = { + ...known, + id: `other-suite.shared-profile.${known.environment.id}.existing-case`, + suite: { ...known.suite, id: "other-suite" }, + profile: { ...known.profile, id: "shared-profile", model: "other-model" }, + }; + const selected = { + ...other, + id: `selected-suite.shared-profile.${known.environment.id}.existing-case`, + suite: { ...known.suite, id: "selected-suite" }, + profile: { ...other.profile, label: "Selected profile", model: "selected-model" }, + }; + const executionId = `selected-suite.shared-profile.${known.environment.id}.new-case`; + const discovered = discoverReportCatalog({ + catalog: [other, selected], + expected: [executionId], + results: [{ ...result(), executionId, suiteId: "selected-suite", profileId: "shared-profile", + environmentId: known.environment.id, caseId: "new-case", provider: selected.profile.provider, model: selected.profile.model }], + }).find(execution => execution.id === executionId); + expect(discovered?.profile).toBe(selected.profile); + expect(discovered?.profile.model).toBe("selected-model"); + }); + + it("matches a new suite's profile by both recorded provider and model", () => { + const known = runnerMatrix[0]!; + const selected = { + ...known, + id: `known-suite.shared-profile.${known.environment.id}.existing-case`, + suite: { ...known.suite, id: "known-suite" }, + profile: { ...known.profile, id: "shared-profile", provider: "selected-provider", model: "selected-model" }, + }; + const executionId = `branch-only-chat.shared-profile.${known.environment.id}.conversation`; + const discovered = discoverReportCatalog({ + catalog: [ + { ...selected, id: `wrong-model.shared-profile.${known.environment.id}.existing-case`, suite: { ...known.suite, id: "wrong-model" }, profile: { ...selected.profile, model: "other-model" } }, + { ...selected, id: `wrong-provider.shared-profile.${known.environment.id}.existing-case`, suite: { ...known.suite, id: "wrong-provider" }, profile: { ...selected.profile, provider: "other-provider" } }, + selected, + ], + expected: [executionId], + results: [{ ...result(), executionId, profileId: "shared-profile", environmentId: known.environment.id, + provider: "selected-provider", model: "selected-model" }], + }).find(execution => execution.id === executionId); + expect(discovered?.profile).toBe(selected.profile); + }); + + it.each([ + null, [], { title: 12 }, { description: false }, { notes: "not an array" }, + { notes: [42] }, { links: "not an array" }, { links: [null] }, + { links: [{ label: "Missing destination" }] }, { links: [{ href: "provenance.json" }] }, + ])("rejects malformed presentation metadata before rewriting retained measurements (%#)", async presentation => { + const bundle = await mkdtemp(path.join(os.tmpdir(), "branch-report-invalid-presentation-")); + temporary.push(bundle); + const file = path.join(bundle, "normalized-results.json"); + const retained = JSON.stringify({ ...campaign(), presentation }); + await writeFile(file, retained); + await expect(regenerateRunnerDashboard({ bundle, historyFile: null })).rejects.toThrow("invalid presentation metadata"); + expect(await readFile(file, "utf8")).toBe(retained); + await expect(readFile(path.join(bundle, "index.html"), "utf8")).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("regenerates branch-only cards and gallery from persisted JSON using trusted code", async () => { const bundle = await mkdtemp( path.join(os.tmpdir(), "branch-report-regenerate-"), @@ -197,10 +259,17 @@ describe("trusted report catalog discovery", () => { await writeFile(path.join(evidence, "final-state.png"), "fixture"); await writeFile( path.join(bundle, "normalized-results.json"), - JSON.stringify({ ...campaign(), suites: [] }), + JSON.stringify({ ...campaign([{ ...result(), targetUrl: "https://qa.example.test", matcherResults: [{ matcher: { kind: "json_path", path: "saved", expected: true }, passed: true, detail: "Saved after reload" }] }]), suites: [], presentation: { title: "QA basics", notes: ["Original measurement retained"], links: [{ label: "Provenance", href: "provenance.json" }, { label: "Unsafe destination", href: "javascript:alert(1)" }] } }), ); await regenerateRunnerDashboard({ bundle, historyFile: null }); const page = await readFile(path.join(bundle, "index.html"), "utf8"); + expect(page).toContain('data-gallery-target-url="https://qa.example.test/"'); + expect(page).toContain('href="https://qa.example.test/" target="_blank"'); + expect(page).toContain("Saved after reload"); + expect(page).toContain("Original measurement retained"); + expect(page).toContain('Provenance'); + expect(page).not.toContain("javascript:alert(1)"); + expect(page).not.toContain("Unsafe destination"); expect(page).toContain(`data-execution-id="${id}"`); expect(page).toContain(`data-execution-id="${missingId}"`); expect(page).toContain( @@ -209,6 +278,8 @@ describe("trusted report catalog discovery", () => { const normalized = JSON.parse( await readFile(path.join(bundle, "normalized-results.json"), "utf8"), ); + expect(normalized.presentation).toEqual({ title: "QA basics", notes: ["Original measurement retained"], links: [{ label: "Provenance", href: "provenance.json" }, { label: "Unsafe destination", href: "javascript:alert(1)" }] }); + expect(normalized.results[0].targetUrl).toBe("https://qa.example.test"); expect(normalized.suites).toEqual([ expect.objectContaining({ suiteId: "branch-only-chat", diff --git a/tests/runner-e2e/report-catalog.ts b/tests/runner-e2e/report-catalog.ts index 3128832ec17..229faa14cea 100644 --- a/tests/runner-e2e/report-catalog.ts +++ b/tests/runner-e2e/report-catalog.ts @@ -98,7 +98,10 @@ export function discoverReportCatalog(input: { expectedMatrixSize: suiteIds.length, }; const knownProfile = input.catalog.find( - (execution) => execution.profile.id === identity.profileId, + (execution) => execution.suite.id === identity.suiteId && execution.profile.id === identity.profileId, + )?.profile ?? input.catalog.find( + (execution) => execution.profile.id === identity.profileId && + (!result || execution.profile.provider === result.provider && execution.profile.model === result.model), )?.profile; const knownEnvironment = input.catalog.find( (execution) => execution.environment.id === identity.environmentId, diff --git a/tests/runner-e2e/report-navigation.test.ts b/tests/runner-e2e/report-navigation.test.ts index 44ecb6cd657..f978550cb4e 100644 --- a/tests/runner-e2e/report-navigation.test.ts +++ b/tests/runner-e2e/report-navigation.test.ts @@ -42,6 +42,9 @@ describe("evaluation report navigation", () => { const history = emptyRunnerHistory(); history.campaigns = [{campaignId:"campaign-a",publicUrl:"/custom/reports/campaigns/campaign-a/"} as typeof history.campaigns[number]]; expect(reportHistoryHref(history,"campaign-a")).toBe("/custom/reports/"); - expect(reportHistoryHref(undefined,"campaign-a")).toBe("../../"); + expect(reportHistoryHref(undefined,"campaign-a")).toBe("https://pages.paperclip.ing/evals/"); + history.campaigns[0]!.publicUrl = "https://pages.paperclip.ing/standalone-report/"; + expect(reportHistoryHref(history,"campaign-a")).toBe("https://pages.paperclip.ing/evals/"); + expect(page).toContain('class="brand-lockup report-brand" href="https://pages.paperclip.ing/evals/"'); }); }); diff --git a/tests/runner-e2e/report-navigation.ts b/tests/runner-e2e/report-navigation.ts index e9d20fd632b..58a86f40c17 100644 --- a/tests/runner-e2e/report-navigation.ts +++ b/tests/runner-e2e/report-navigation.ts @@ -10,7 +10,9 @@ export function reportHtml(value: unknown) { export function reportHistoryHref(history: RunnerE2EHistoryIndex | undefined, campaignId: string | undefined) { const url = history?.campaigns.find(campaign => campaign.campaignId === campaignId)?.publicUrl; - return url?.replace(/campaigns\/[^/]+\/(?:index\.html)?$/, "") ?? "../../"; + return url && /campaigns\/[^/]+\/(?:index\.html)?$/.test(url) + ? url.replace(/campaigns\/[^/]+\/(?:index\.html)?$/, "") + : "https://pages.paperclip.ing/evals/"; } export function renderReportShell(input: { @@ -30,7 +32,7 @@ export function renderReportShell(input: { return `