diff --git a/doc/evals.md b/doc/evals.md index 6cb2ce4b3da..5dad409c2b6 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -331,6 +331,14 @@ compatible complete campaigns. Refreshing an existing report from retained evidence has zero provider calls and is a new presentation of the old measurement, not a new model run. +Product E2E test details and screenshot galleries show the recorded application +URL when it is available. The worker environment, such as Daytona, does not +imply an application URL. Public projections can retain the reviewed URL in +`targetUrl`. Test details put recorded checks first and keep raw usage, timing, +and startup evidence under Diagnostics. Optional `presentation` metadata in a +retained campaign supplies its title, description, notes, and evidence links +inside the shared report shell when the dashboard is regenerated. + Existing public histories are available at [Runner protocol history](https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/index.html) and [Runner Product E2E history](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/). diff --git a/tests/runner-e2e/dashboard-regenerate.ts b/tests/runner-e2e/dashboard-regenerate.ts index 2913bd96459..98bba7f8d45 100644 --- a/tests/runner-e2e/dashboard-regenerate.ts +++ b/tests/runner-e2e/dashboard-regenerate.ts @@ -1,7 +1,7 @@ import path from "node:path"; import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; import { runnerMatrix } from "./catalog.js"; -import { renderRunnerE2EDashboard } from "./dashboard.js"; +import { renderRunnerE2EDashboard, type RunnerDashboardInput } from "./dashboard.js"; import { campaignFeedbackReviewText } from "./feedback-dispositions.js"; import { buildRunnerCampaign, @@ -30,6 +30,25 @@ interface PublishedCampaign { operations?: RunnerE2ECampaign["operations"]; feedbackReview?: RunnerE2ECampaign["feedbackReview"]; braintrustExperiment?: RunnerE2ECampaign["braintrustExperiment"]; + presentation?: RunnerDashboardInput["presentation"]; +} + +function validatePresentation(value: unknown) { + if (value === undefined) return; + const invalid = () => { + throw new Error("Published bundle has invalid presentation metadata in normalized-results.json"); + }; + if (!value || typeof value !== "object" || Array.isArray(value)) invalid(); + const presentation = value as Record; + for (const field of ["title", "description"]) { + if (presentation[field] !== undefined && typeof presentation[field] !== "string") invalid(); + } + if (presentation.notes !== undefined && + (!Array.isArray(presentation.notes) || presentation.notes.some(note => typeof note !== "string"))) invalid(); + if (presentation.links !== undefined && + (!Array.isArray(presentation.links) || presentation.links.some(link => + !link || typeof link !== "object" || Array.isArray(link) || + typeof link.label !== "string" || typeof link.href !== "string"))) invalid(); } async function relativeFiles(root: string, current = root): Promise { @@ -89,6 +108,7 @@ export async function regenerateRunnerDashboard(input: { ) { throw new Error("Published bundle has an invalid normalized-results.json"); } + validatePresentation(normalized.presentation); const results = normalized.results.map(upgradeRunnerResult); const expected = normalized.expected.map(canonicalExecutionId); @@ -162,16 +182,18 @@ export async function regenerateRunnerDashboard(input: { }), ); const dashboard = renderRunnerE2EDashboard({ - title: "Runner Full-Stack E2E", + title: normalized.presentation?.title ?? "Runner Full-Stack E2E", generatedAt: normalized.generatedAt, expected, catalog: runnerMatrix, entries, campaign, history, + presentation: normalized.presentation, }); const upgraded = { ...campaign, + ...(normalized.presentation ? { presentation: normalized.presentation } : {}), results: campaign.results.map((result, index) => ({ ...result, evidenceValid: diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index b8cb4ac92b0..e8ff48e7705 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -2,12 +2,13 @@ import { campaignAttemptTimeLabel, campaignTimingScope, renderWorkerPhaseTiming, import { html, durationLabel, tokenLabel, llmCostLabel, usdLabel } from "./report-format.js"; import { renderReportShell, reportNavigationStyles, reportShellScript } from "./report-navigation.js"; import { reportBrowserScript } from "./report-browser.js"; +import { reportEvidenceHref, reportTargetUrl } from "./report-target.js"; import { isIncompleteFirstTaskResult, renderCaseOutcome } from "./case-outcome.js"; import { latencyIntervalLabel } from "./latency-timing.js"; import { campaignSuiteSummaryStyles, renderCampaignSuiteSummary, renderSuiteCaseIndex } from "./campaign-summary.js"; import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js"; import { journeyFeatureStyles, renderJourneyFeatureCoverage, renderJourneyFeatureOptions } from "./feature-coverage.js"; -import { STARTUP_REUSE_LAYERS, summarizeStartupReuse } from "../../packages/adapter-utils/src/startup-reuse.js"; +import { STARTUP_REUSE_LAYERS } from "../../packages/adapter-utils/src/startup-reuse.js"; import { renderFirstTaskDetails } from "./first-task-report.js"; import { summarizeCampaignFeedback } from "./feedback.js"; import { renderCampaignFeedbackReview } from "./feedback-dispositions.js"; @@ -45,6 +46,12 @@ export interface RunnerDashboardInput { entries: readonly RunnerDashboardEntry[]; campaign?: RunnerE2ECampaign; history?: RunnerE2EHistoryIndex; + presentation?: { + title?: string; + description?: string; + notes?: readonly string[]; + links?: readonly { label: string; href: string }[]; + }; } interface ResolvedScreenshot { @@ -147,6 +154,7 @@ function renderCase( : item, ); const billing = entry ? summarizeExecutionBilling(entry.result) : null; + const targetUrl = reportTargetUrl(entry?.result); const capabilities = journeyCapabilities(execution); const features = [...new Map(capabilities.map((capability) => [capability.feature.id, capability.feature])).values()]; const outcome = entry ? renderCaseOutcome(entry.result, entry.valid, entry.errors) : ""; @@ -233,9 +241,9 @@ function renderCase( ${html(latencyIntervalLabel(probe))}${html(`${probe.target.operator} ${probe.target.ms} ms`)} ${html(probe.resolutionMs === null ? "unknown" : `${probe.resolutionMs} ms`)} `).join(""); - const startupReuseRows = (entry?.result.startupReuse ?? (entry?.result.runIds ?? []).map(runId => ({ - runId, ...summarizeStartupReuse([]), - }))).map(reuse => ` + const startupReuseRows = (entry?.result.startupReuse ?? []).filter(reuse => + reuse.providerLaunchCount !== null || STARTUP_REUSE_LAYERS.some(layer => reuse.layers[layer].state !== "unknown"), + ).map(reuse => ` ${html(reuse.runId)}${html(reuse.warmth)} ${STARTUP_REUSE_LAYERS.map(layer => `${html(reuse.layers[layer].state)} (${html(reuse.layers[layer].reason)})`).join("")} ${html(reuse.providerLaunchCount === null ? "unknown" : reuse.providerLaunchCount)} @@ -250,6 +258,7 @@ function renderCase( data-gallery-href="${html(item.href)}" data-gallery-label="${html(item.label)}" data-gallery-execution="${html(execution.id)}" + data-gallery-target-url="${html(targetUrl)}" data-gallery-case="${html(execution.task.label)}" data-gallery-profile="${html(execution.profile.label)}" data-gallery-generation="${html(execution.profile.generation)}" @@ -262,7 +271,7 @@ function renderCase( data-gallery-status="${html(label)}" data-gallery-duration="${html(entry ? durationLabel(entry.result.durationMs) : "Not run")}" data-gallery-tokens="${html(billing ? `${tokenLabel(billing.llm.inputTokens, billing.llm)} in · ${tokenLabel(billing.llm.outputTokens, billing.llm)} out` : "Unavailable")}" - data-gallery-matchers="${html(matcherResults.length > 0 ? `${passedMatchers}/${matcherResults.length} matchers passed${notReached ? ` · ${notReached} not reached` : ""}` : "No matchers recorded")}" + data-gallery-matchers="${html(matcherResults.length > 0 ? `${passedMatchers}/${matcherResults.length} checks passed${notReached ? ` · ${notReached} not reached` : ""}` : "No checks recorded")}" aria-label="Open ${html(item.label)} for ${html(execution.id)} in gallery" > ${html(item.label)} for ${html(execution.id)} @@ -276,7 +285,7 @@ function renderCase(
Tokens${html(tokenLabel(billing.llm.inputTokens, billing.llm))} in · ${html(tokenLabel(billing.llm.outputTokens, billing.llm))} out${html(tokenLabel(billing.llm.cachedInputTokens, billing.llm))} cached · ${billing.llm.runsWithTokenUsage}/${billing.llm.runCount} runs covered
LLM spend${html(llmCostLabel(billing.reportedCostUsd, billing.llm))}${billing.llm.runsWithReportedCost}/${billing.llm.runCount} runs provider-priced
${(billing.llm.runsWithEstimatedCost ?? 0) > 0 ? `
LLM estimate${html(billingCoverageLabel(usdLabel(billing.llm.estimatedCostUsd ?? 0), billing.llm.runsWithEstimatedCost ?? 0, billing.llm.runCount))}Estimated from captured usage
` : ""} -
Execution${billing.runtime.estimatedListCostUsd === undefined ? html(billing.runtime.costStatus === "not_metered" ? "Local · not metered" : "Cost unavailable") : `${html(usdLabel(billing.runtime.estimatedListCostUsd))} est.`}${html(durationLabel(billing.runtime.agentRunDurationMs))} agent${billing.runtime.leaseDurationMs === null ? "" : ` · ${html(durationLabel(billing.runtime.leaseDurationMs))} lease`}
+
Execution${billing.runtime.estimatedListCostUsd === undefined ? html(billing.runtime.costStatus === "not_metered" ? "Local · not metered" : "Unavailable") : `${html(usdLabel(billing.runtime.estimatedListCostUsd))} est.`}${billing.runtime.agentRunDurationMs === null && billing.runtime.leaseDurationMs === null ? "" : `${billing.runtime.agentRunDurationMs === null ? "" : `${html(durationLabel(billing.runtime.agentRunDurationMs))} agent`}${billing.runtime.leaseDurationMs === null ? "" : ` · ${html(durationLabel(billing.runtime.leaseDurationMs))} lease`}`}
${billing.assistant ? `
External assistant${html(usdLabel(billing.assistant.estimatedCostUsd))} est.${html(billing.assistant.model)} · ${billing.assistant.requests} requests · ${html(tokenLabel(billing.assistant.inputTokens + billing.assistant.cachedInputTokens))} in / ${html(tokenLabel(billing.assistant.outputTokens))} out
` : ""} ` : ""; @@ -306,13 +315,22 @@ function renderCase( ${gallery}
Test details${matcherResults.length ? `${passedMatchers}/${matcherResults.length} checks${notReached ? ` · ${notReached} not reached` : ""}` : "No checks recorded"}${entry ? ` · ${html(durationLabel(entry.result.durationMs))}` : ""} - ${passed ? outcome : ""} + ${entry ? `
+
Tested URL
${targetUrl ? `${html(targetUrl)} ` : "Not recorded"}
+
Environment
${html(execution.environment.label)}
+ ${entry.result.model ? `
Model
${html(entry.result.model)}
` : ""} +
Runtime
${html(entry.result.runtimeMode === "none" ? "Browser only" : `${entry.result.runtimeMode} · ${entry.result.provider ?? execution.profile.provider}`)}
+
` : ""} + ${matcherResults.length || notReached ? `
    ${matcherResults.map(result => `
  • ${result.passed ? "Pass" : "Fail"}${html(result.detail)}
    • `).join("")}${firstTaskChecks?.filter(check => check.notReached).map(check => `
  • Not reached${html(check.detail)} · ${html(check.notReached)}
    • `).join("") ?? ""}
` : ""} ${billingStrip} + ${renderFirstTaskDetails(entry?.result)} + ${links} +
+ Diagnostics + ${passed ? outcome : ""} ${entry ? `

Worker phase timing: ${entry.result.workerTiming ? html(entry.result.workerTiming.status === "unavailable" ? `unavailable (${entry.result.workerTiming.reason})` : entry.result.workerTiming.status) : "not captured"}. ${entry.result.workerTiming?.status === "partial" ? "Only completed phases have observed durations." : ""}

` : ""} ${entry?.result.feedback ? `

Feedback: ${entry.result.feedback.complaints} complaints · ${entry.result.feedback.suggestions} suggestions · ${entry.result.feedback.review.length} distinct items; see campaign feedback review. Saved-feedback collection: ${html(entry.result.feedback.collection)}; invocation coverage: ${html(entry.result.feedback.invocationCollection)}. Tool receipts: ${entry.result.feedback.invocations.failed} failed · ${entry.result.feedback.invocations.unknown} unresolved. Full bodies: access-controlled snapshots/agent-feedback.json or snapshots/agent-feedback-recovered.json.

` : entry ? `

Feedback was not captured for this attempt.

` : ""} ${entry?.result.tracing ? `

Braintrust worker export: ${html(entry.result.tracing.status)}; native configuration: ${html(entry.result.tracing.nativeConfiguration)}. ${entry.result.tracing.finalization ? `Supervisor outcome: ${html(entry.result.tracing.finalization.outcome)}; finalization export: ${html(entry.result.tracing.finalization.status)}.` : "Supervisor finalization was not captured."} Trace ${html(entry.result.tracing.traceId)} in project ${html(entry.result.tracing.projectId)}. Indexing and native-span correlation require independent verification.

` : ""} - ${links} - ${renderFirstTaskDetails(entry?.result)} ${html(execution.id)} ${capabilities.length ? `
    ${capabilities.map((capability) => `
  • ${html(capability.feature.title)} · ${html(capability.label)}
    • `).join("")}
` : ""} ${ @@ -329,12 +347,13 @@ function renderCase( ` : "" } -

${html(detail)}

+ ${entry?.errors.length ? `

${html(detail)}

` : ""} ${turnTimingRows ? `
${turnTimingRows}
TurnRunLeaseSchedulerRun durationFinal settlement (legacy response field)
` : ""} ${latencyRows ? `
${latencyRows}
EventTiming verdictPossible latencyTargetResolution
` : ""} ${startupReuseRows ? `
${startupReuseRows}
Startup reuse evidence
RunWarmthEnvironment/leaseWorkspaceRunnerProvider processProvider sessionProvider launches
` : ""} - ${matcherRows ? `
${matcherRows}
ResultMatcherExpectationDetail
` : `

No matcher result was recorded.

`} - ${entry ? `
Usage, billing and timing metadata
${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null, workerTiming: entry.result.workerTiming ?? null }, null, 2))}
` : ""} + ${matcherRows ? `
${matcherRows}
ResultMatcherExpectationDetail
` : ""} + ${entry ? `
Usage, billing and timing metadata
${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null, workerTiming: entry.result.workerTiming ?? null, startupReuse: entry.result.startupReuse ?? null }, null, 2))}
` : ""} +
`; } @@ -379,6 +398,10 @@ function renderSuiteResults(input: { } export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { + const presentationLinks = (input.presentation?.links ?? []).flatMap(link => { + const href = reportEvidenceHref(link.href); + return href ? [{ ...link, href }] : []; + }); const expected = new Set(input.expected); const catalog = discoverReportCatalog({ catalog: input.catalog, @@ -662,6 +685,17 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .case-diagnostics > summary::before { content: "›"; color: var(--muted-foreground); font-size: 16px; line-height: 1; } .case-diagnostics[open] > summary::before { transform: rotate(90deg); } .case-detail-summary { margin-left: auto; color: var(--muted-foreground); font: 400 11px/1.5 var(--font-mono); } + .case-facts { display: grid; gap: 10px; margin: 18px 0; font-size: 12px; } + .case-facts > div { display: grid; grid-template-columns: 92px minmax(0, 1fr); align-items: baseline; gap: 12px; } + .case-facts dd { min-width: 0; overflow-wrap: anywhere; font: inherit; } + .case-facts a { text-underline-offset: 3px; } + .case-checks { display: grid; gap: 9px; list-style: none; padding: 14px 0; margin: 0; border-top: 1px solid var(--border); font-size: 12px; } + .case-checks li { display: flex; align-items: baseline; gap: 10px; overflow-wrap: anywhere; } + .check-state { flex: none; min-width: 32px; color: var(--muted-foreground); font-size: 10px; font-weight: 600; } + .check-passed { color: var(--pass-text); } + .check-failed { color: var(--fail-text); } + .case-advanced { margin-top: 18px; } + .case-advanced > summary { color: var(--muted-foreground); font-weight: 400; } .case-capabilities { padding-left: 16px; color: var(--muted-foreground); font-size: 11px; } .case-outcome { margin: 12px 0; padding: 12px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--raised); font-size: 12px; } .case-incomplete > .case-outcome { border-color: var(--missing-border); } @@ -735,6 +769,8 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .gallery-context span { display: inline-flex; align-items: baseline; gap: 6px; min-width: 0; } .gallery-context strong { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-size: 12px; } .gallery-context em { color: #b8b8b5; font: 10px/1.4 var(--font-mono); font-style: normal; white-space: nowrap; } + .gallery-context .gallery-target { flex-basis: 100%; } + .gallery-target a { color: #fafafa; font-size: 12px; overflow-wrap: anywhere; text-underline-offset: 3px; } .gallery-facts { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 18px; margin-top: 8px; } .gallery-facts > span { display: inline-flex; align-items: baseline; gap: 6px; min-width: 0; } .gallery-facts em { color: #b8b8b5; font: 10px/1.4 var(--font-mono); font-style: normal; text-transform: uppercase; } @@ -833,8 +869,8 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {

Full-stack acceptance campaign

-

${html(input.title)}

-

Results across runner profiles and execution environments. Declared PNG screenshots and normalized results are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.

+

${html(input.presentation?.title ?? input.title)}

+

${html(input.presentation?.description ?? "Results across runner profiles and execution environments. Declared PNG screenshots and normalized results are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.")}

${input.campaign ? `

${html(input.generatedAt)}${input.campaign.source.sha ? `${html(input.campaign.source.sha.slice(0,8))}` : ""}${input.campaign.source.workflowRunUrl ? `Source run ↗` : ""}

` : ""}
@@ -847,6 +883,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
+ ${input.presentation?.notes?.length || presentationLinks.length ? `
About this run${input.presentation?.notes?.map(note => `

${html(note)}

`).join("") ?? ""}${presentationLinks.length ? `` : ""}
` : ""} ${renderCampaignSuiteSummary(input.campaign?.suites ?? [], catalog, "", false)}
Campaign usage and spend
@@ -894,6 +931,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { Environment: + Tested URL:
@@ -953,6 +991,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { const profileDetail = dialog.querySelector("[data-gallery-profile-detail]"); const environment = dialog.querySelector("[data-gallery-environment]"); const environmentDetail = dialog.querySelector("[data-gallery-environment-detail]"); + const targetUrl = dialog.querySelector("[data-gallery-target-url]"); const caseLabel = dialog.querySelector("[data-gallery-case]"); const runtime = dialog.querySelector("[data-gallery-runtime]"); const status = dialog.querySelector("[data-gallery-status]"); @@ -983,6 +1022,9 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { profileDetail.textContent = item.dataset.galleryGeneration + " · " + item.dataset.galleryProvider + " · " + item.dataset.galleryModel; environment.textContent = item.dataset.galleryEnvironment; environmentDetail.textContent = "Provider: " + item.dataset.galleryEnvironmentProvider + " · Target: " + item.dataset.galleryExecutionTarget; + targetUrl.textContent = item.dataset.galleryTargetUrl || "Not recorded"; + if (item.dataset.galleryTargetUrl) targetUrl.href = item.dataset.galleryTargetUrl; + else targetUrl.removeAttribute("href"); caseLabel.textContent = item.dataset.galleryCase; runtime.textContent = item.dataset.galleryRuntime + " runtime"; status.textContent = item.dataset.galleryStatus; diff --git a/tests/runner-e2e/first-task-report.test.ts b/tests/runner-e2e/first-task-report.test.ts index 1150d002338..974f1ca8b6d 100644 --- a/tests/runner-e2e/first-task-report.test.ts +++ b/tests/runner-e2e/first-task-report.test.ts @@ -265,7 +265,7 @@ describe("first-task conversation report", () => { }); const page = render(); expect(page).toContain('data-gallery-status="incomplete"'); - expect(page).toContain('data-gallery-matchers="1/1 matchers passed · 1 not reached"'); + expect(page).toContain('data-gallery-matchers="1/1 checks passed · 1 not reached"'); expect(page).toContain('data-gallery-label="Task state when recording stopped"'); expect(page).toContain("Incomplete journey"); expect(page).toContain('1Incomplete'); diff --git a/tests/runner-e2e/report-catalog.test.ts b/tests/runner-e2e/report-catalog.test.ts index 5cd96f1e078..0cb16af378c 100644 --- a/tests/runner-e2e/report-catalog.test.ts +++ b/tests/runner-e2e/report-catalog.test.ts @@ -187,6 +187,68 @@ describe("trusted report catalog discovery", () => { ).toThrow("must be unique"); }); + it("keeps the suite's profile when another suite uses the same profile ID", () => { + const known = runnerMatrix[0]!; + const other = { + ...known, + id: `other-suite.shared-profile.${known.environment.id}.existing-case`, + suite: { ...known.suite, id: "other-suite" }, + profile: { ...known.profile, id: "shared-profile", model: "other-model" }, + }; + const selected = { + ...other, + id: `selected-suite.shared-profile.${known.environment.id}.existing-case`, + suite: { ...known.suite, id: "selected-suite" }, + profile: { ...other.profile, label: "Selected profile", model: "selected-model" }, + }; + const executionId = `selected-suite.shared-profile.${known.environment.id}.new-case`; + const discovered = discoverReportCatalog({ + catalog: [other, selected], + expected: [executionId], + results: [{ ...result(), executionId, suiteId: "selected-suite", profileId: "shared-profile", + environmentId: known.environment.id, caseId: "new-case", provider: selected.profile.provider, model: selected.profile.model }], + }).find(execution => execution.id === executionId); + expect(discovered?.profile).toBe(selected.profile); + expect(discovered?.profile.model).toBe("selected-model"); + }); + + it("matches a new suite's profile by both recorded provider and model", () => { + const known = runnerMatrix[0]!; + const selected = { + ...known, + id: `known-suite.shared-profile.${known.environment.id}.existing-case`, + suite: { ...known.suite, id: "known-suite" }, + profile: { ...known.profile, id: "shared-profile", provider: "selected-provider", model: "selected-model" }, + }; + const executionId = `branch-only-chat.shared-profile.${known.environment.id}.conversation`; + const discovered = discoverReportCatalog({ + catalog: [ + { ...selected, id: `wrong-model.shared-profile.${known.environment.id}.existing-case`, suite: { ...known.suite, id: "wrong-model" }, profile: { ...selected.profile, model: "other-model" } }, + { ...selected, id: `wrong-provider.shared-profile.${known.environment.id}.existing-case`, suite: { ...known.suite, id: "wrong-provider" }, profile: { ...selected.profile, provider: "other-provider" } }, + selected, + ], + expected: [executionId], + results: [{ ...result(), executionId, profileId: "shared-profile", environmentId: known.environment.id, + provider: "selected-provider", model: "selected-model" }], + }).find(execution => execution.id === executionId); + expect(discovered?.profile).toBe(selected.profile); + }); + + it.each([ + null, [], { title: 12 }, { description: false }, { notes: "not an array" }, + { notes: [42] }, { links: "not an array" }, { links: [null] }, + { links: [{ label: "Missing destination" }] }, { links: [{ href: "provenance.json" }] }, + ])("rejects malformed presentation metadata before rewriting retained measurements (%#)", async presentation => { + const bundle = await mkdtemp(path.join(os.tmpdir(), "branch-report-invalid-presentation-")); + temporary.push(bundle); + const file = path.join(bundle, "normalized-results.json"); + const retained = JSON.stringify({ ...campaign(), presentation }); + await writeFile(file, retained); + await expect(regenerateRunnerDashboard({ bundle, historyFile: null })).rejects.toThrow("invalid presentation metadata"); + expect(await readFile(file, "utf8")).toBe(retained); + await expect(readFile(path.join(bundle, "index.html"), "utf8")).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("regenerates branch-only cards and gallery from persisted JSON using trusted code", async () => { const bundle = await mkdtemp( path.join(os.tmpdir(), "branch-report-regenerate-"), @@ -197,10 +259,17 @@ describe("trusted report catalog discovery", () => { await writeFile(path.join(evidence, "final-state.png"), "fixture"); await writeFile( path.join(bundle, "normalized-results.json"), - JSON.stringify({ ...campaign(), suites: [] }), + JSON.stringify({ ...campaign([{ ...result(), targetUrl: "https://qa.example.test", matcherResults: [{ matcher: { kind: "json_path", path: "saved", expected: true }, passed: true, detail: "Saved after reload" }] }]), suites: [], presentation: { title: "QA basics", notes: ["Original measurement retained"], links: [{ label: "Provenance", href: "provenance.json" }, { label: "Unsafe destination", href: "javascript:alert(1)" }] } }), ); await regenerateRunnerDashboard({ bundle, historyFile: null }); const page = await readFile(path.join(bundle, "index.html"), "utf8"); + expect(page).toContain('data-gallery-target-url="https://qa.example.test/"'); + expect(page).toContain('href="https://qa.example.test/" target="_blank"'); + expect(page).toContain("Saved after reload"); + expect(page).toContain("Original measurement retained"); + expect(page).toContain('Provenance'); + expect(page).not.toContain("javascript:alert(1)"); + expect(page).not.toContain("Unsafe destination"); expect(page).toContain(`data-execution-id="${id}"`); expect(page).toContain(`data-execution-id="${missingId}"`); expect(page).toContain( @@ -209,6 +278,8 @@ describe("trusted report catalog discovery", () => { const normalized = JSON.parse( await readFile(path.join(bundle, "normalized-results.json"), "utf8"), ); + expect(normalized.presentation).toEqual({ title: "QA basics", notes: ["Original measurement retained"], links: [{ label: "Provenance", href: "provenance.json" }, { label: "Unsafe destination", href: "javascript:alert(1)" }] }); + expect(normalized.results[0].targetUrl).toBe("https://qa.example.test"); expect(normalized.suites).toEqual([ expect.objectContaining({ suiteId: "branch-only-chat", diff --git a/tests/runner-e2e/report-catalog.ts b/tests/runner-e2e/report-catalog.ts index 3128832ec17..229faa14cea 100644 --- a/tests/runner-e2e/report-catalog.ts +++ b/tests/runner-e2e/report-catalog.ts @@ -98,7 +98,10 @@ export function discoverReportCatalog(input: { expectedMatrixSize: suiteIds.length, }; const knownProfile = input.catalog.find( - (execution) => execution.profile.id === identity.profileId, + (execution) => execution.suite.id === identity.suiteId && execution.profile.id === identity.profileId, + )?.profile ?? input.catalog.find( + (execution) => execution.profile.id === identity.profileId && + (!result || execution.profile.provider === result.provider && execution.profile.model === result.model), )?.profile; const knownEnvironment = input.catalog.find( (execution) => execution.environment.id === identity.environmentId, diff --git a/tests/runner-e2e/report-navigation.test.ts b/tests/runner-e2e/report-navigation.test.ts index 44ecb6cd657..f978550cb4e 100644 --- a/tests/runner-e2e/report-navigation.test.ts +++ b/tests/runner-e2e/report-navigation.test.ts @@ -42,6 +42,9 @@ describe("evaluation report navigation", () => { const history = emptyRunnerHistory(); history.campaigns = [{campaignId:"campaign-a",publicUrl:"/custom/reports/campaigns/campaign-a/"} as typeof history.campaigns[number]]; expect(reportHistoryHref(history,"campaign-a")).toBe("/custom/reports/"); - expect(reportHistoryHref(undefined,"campaign-a")).toBe("../../"); + expect(reportHistoryHref(undefined,"campaign-a")).toBe("https://pages.paperclip.ing/evals/"); + history.campaigns[0]!.publicUrl = "https://pages.paperclip.ing/standalone-report/"; + expect(reportHistoryHref(history,"campaign-a")).toBe("https://pages.paperclip.ing/evals/"); + expect(page).toContain('class="brand-lockup report-brand" href="https://pages.paperclip.ing/evals/"'); }); }); diff --git a/tests/runner-e2e/report-navigation.ts b/tests/runner-e2e/report-navigation.ts index e9d20fd632b..58a86f40c17 100644 --- a/tests/runner-e2e/report-navigation.ts +++ b/tests/runner-e2e/report-navigation.ts @@ -10,7 +10,9 @@ export function reportHtml(value: unknown) { export function reportHistoryHref(history: RunnerE2EHistoryIndex | undefined, campaignId: string | undefined) { const url = history?.campaigns.find(campaign => campaign.campaignId === campaignId)?.publicUrl; - return url?.replace(/campaigns\/[^/]+\/(?:index\.html)?$/, "") ?? "../../"; + return url && /campaigns\/[^/]+\/(?:index\.html)?$/.test(url) + ? url.replace(/campaigns\/[^/]+\/(?:index\.html)?$/, "") + : "https://pages.paperclip.ing/evals/"; } export function renderReportShell(input: { @@ -30,7 +32,7 @@ export function renderReportShell(input: { return `