import { html, durationLabel, tokenLabel, llmCostLabel, usdLabel } from "./report-format.js"; import { renderReportShell, reportNavigationStyles, reportShellScript } from "./report-navigation.js"; import { reportBrowserScript } from "./report-browser.js"; import { isIncompleteFirstTaskResult, renderCaseOutcome } from "./case-outcome.js"; import { latencyIntervalLabel } from "./latency-timing.js"; import { campaignSuiteSummaryStyles, renderCampaignSuiteSummary, renderSuiteCaseIndex } from "./campaign-summary.js"; import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js"; import { journeyFeatureStyles, renderJourneyFeatureCoverage, renderJourneyFeatureOptions } from "./feature-coverage.js"; import { STARTUP_REUSE_LAYERS, summarizeStartupReuse } from "../../packages/adapter-utils/src/startup-reuse.js"; import { renderFirstTaskDetails } from "./first-task-report.js"; import { discoverReportCatalog, type ReportExecution, } from "./report-catalog.js"; import type { RunnerE2ECampaign, RunnerE2EHistoryIndex, RunnerE2EResult, RunnerE2ESuiteSummary, } from "./types.js"; import { aggregateCampaignBilling, summarizeExecutionBilling, } from "./billing.js"; export interface RunnerDashboardEntry { result: RunnerE2EResult; valid: boolean; errors: readonly string[]; evidenceBaseHref?: string; evidenceFiles?: readonly string[]; } export interface RunnerDashboardInput { title: string; generatedAt: string; expected: readonly string[]; catalog: readonly ReportExecution[]; entries: readonly RunnerDashboardEntry[]; campaign?: RunnerE2ECampaign; history?: RunnerE2EHistoryIndex; } interface ResolvedScreenshot { id: string; label: string; file: string; href: string; } function safeEvidenceHref(base: string | undefined, relative: string) { if (!base || /^(?:[a-z]+:|\/\/)/i.test(base)) return null; const cleanBase = base .split("/") .filter((segment) => segment && segment !== "." && segment !== "..") .map(encodeURIComponent) .join("/"); const cleanRelative = relative .split("/") .filter((segment) => segment && segment !== "." && segment !== "..") .map(encodeURIComponent) .join("/"); return `${cleanBase}/${cleanRelative}`; } function compactJson(value: unknown) { const serialized = JSON.stringify(value); return serialized && serialized.length > 1_200 ? `${serialized.slice(0, 1_200)}…` : serialized; } function resolveScreenshots( entry: RunnerDashboardEntry | undefined, ): ResolvedScreenshot[] { const declaredScreenshots = entry?.result.screenshots?.length ? entry.result.screenshots : entry ? [ { id: "final-state", label: "Final visible task state", file: "final-state.png", }, ] : []; const availableFiles = entry?.evidenceFiles ? new Set(entry.evidenceFiles) : null; const screenshots = declaredScreenshots .filter((item) => !availableFiles || availableFiles.has(item.file)) .flatMap((item) => { const href = safeEvidenceHref(entry?.evidenceBaseHref, item.file); return href ? [{ ...item, href }] : []; }); if ( entry?.result.status === "failed" && !screenshots.some((item) => item.file === "failure.png") && (!availableFiles || availableFiles.has("failure.png")) ) { const href = safeEvidenceHref(entry.evidenceBaseHref, "failure.png"); if (href) { screenshots.push({ id: "failure", label: "Failure state", file: "failure.png", href, }); } } return screenshots; } function renderCase( execution: ReportExecution, expected: ReadonlySet, entryById: ReadonlyMap, ) { const selected = expected.has(execution.id); const entry = entryById.get(execution.id); const state = !selected ? "not-selected" : !entry ? "missing" : entry.valid ? "passed" : isIncompleteFirstTaskResult(entry.result, entry.errors) ? "incomplete" : "failed"; const label = state.replace("-", " "); const detail = entry?.errors.join("; ") || (entry?.valid ? "All invariants passed" : selected ? "No result artifact was uploaded" : "Not selected"); const screenshots = resolveScreenshots(entry).map((item) => state === "incomplete" && /failure/i.test(item.label) ? { ...item, label: "Task state when recording stopped" } : item, ); const billing = entry ? summarizeExecutionBilling(entry.result) : null; const capabilities = journeyCapabilities(execution); const features = [...new Map(capabilities.map((capability) => [capability.feature.id, capability.feature])).values()]; const outcome = entry ? renderCaseOutcome(entry.result, entry.valid, entry.errors) : ""; const passed = state === "passed" && entry?.result.status === "passed"; const firstTaskChecks = entry?.result.firstTask?.checks; const notReached = firstTaskChecks?.filter((c) => c.notReached).length ?? 0; const matcherResults = firstTaskChecks?.length ? firstTaskChecks.filter((c) => !c.notReached).map((c) => ({ matcher: { kind: "json_path" as const, path: `firstTask.checks.${c.id}`, expected: true }, passed: c.passed, detail: c.detail, })) : entry?.result.matcherResults ?? []; const passedMatchers = matcherResults.filter( (result) => result.passed, ).length; const searchText = [ execution.id, execution.task.label, execution.profile.label, execution.profile.generation, execution.profile.provider, execution.profile.model, execution.environment.label, execution.environment.provider, execution.suite.label, ...capabilities.flatMap((capability) => [capability.id, capability.label, capability.feature.title]), label, ] .join(" ") .toLowerCase(); const availableFiles = entry?.evidenceFiles ? new Set(entry.evidenceFiles) : null; const playwright = !availableFiles || availableFiles.has("html-report/index.html") ? safeEvidenceHref(entry?.evidenceBaseHref, "html-report/index.html") : null; const structuredLinks = [ ["snapshots/first-task.json", "Full conversation and instruction JSON"], ["snapshots/first-task-run-evidence.json", "Full run / tool-event JSON"], ["regraded-result.json", "Regraded result JSON"], ["result.json", availableFiles?.has("regraded-result.json") ? "Original result JSON" : "Complete result JSON"], ].flatMap(([file, label]) => { if (!availableFiles?.has(file)) return []; const href = safeEvidenceHref(entry?.evidenceBaseHref, file); return href ? [`${html(label)}`] : []; }); const links = playwright || structuredLinks.length > 0 ? `` : ""; const matcherRows = matcherResults .map( (result) => ` ${result.passed ? "Pass" : "Fail"} ${html(result.matcher.kind)} ${html(compactJson(result.matcher))} ${html(result.detail)} `, ) .join(""); const turnTimingRows = (entry?.result.turnTimings ?? []) .map( (timing) => ` ${timing.turn} ${html(timing.runId)} ${html(timing.leaseAcquisitionOutcome)} ${html(timing.schedulerLatencyMs === null ? "unavailable" : durationLabel(timing.schedulerLatencyMs))} ${html(timing.runDurationMs === null ? "unavailable" : durationLabel(timing.runDurationMs))} ${html(timing.responseLatencyMs === null ? "unavailable" : durationLabel(timing.responseLatencyMs))} `, ) .join(""); const latencyRows = (entry?.result.latencyProbes ?? []).map(probe => ` ${html(probe.event)}${html(probe.verdict)} ${html(latencyIntervalLabel(probe))}${html(`${probe.target.operator} ${probe.target.ms} ms`)} ${html(probe.resolutionMs === null ? "unknown" : `${probe.resolutionMs} ms`)} `).join(""); const startupReuseRows = (entry?.result.startupReuse ?? (entry?.result.runIds ?? []).map(runId => ({ runId, ...summarizeStartupReuse([]), }))).map(reuse => ` ${html(reuse.runId)}${html(reuse.warmth)} ${STARTUP_REUSE_LAYERS.map(layer => `${html(reuse.layers[layer].state)} (${html(reuse.layers[layer].reason)})`).join("")} ${html(reuse.providerLaunchCount === null ? "unknown" : reuse.providerLaunchCount)} `).join(""); const gallery = screenshots.length ? `` : ""; const billingStrip = billing ? `
Tokens${html(tokenLabel(billing.llm.inputTokens, billing.llm))} in · ${html(tokenLabel(billing.llm.outputTokens, billing.llm))} out${html(tokenLabel(billing.llm.cachedInputTokens, billing.llm))} cached · ${billing.llm.runsWithTokenUsage}/${billing.llm.runCount} runs covered
LLM spend${html(llmCostLabel(billing.reportedCostUsd, billing.llm))}${billing.llm.runsWithReportedCost}/${billing.llm.runCount} runs provider-priced
Execution${billing.runtime.estimatedListCostUsd === undefined ? html(billing.runtime.costStatus === "not_metered" ? "Local · not metered" : "Cost unavailable") : `${html(usdLabel(billing.runtime.estimatedListCostUsd))} est.`}${html(durationLabel(billing.runtime.agentRunDurationMs))} agent${billing.runtime.leaseDurationMs === null ? "" : ` · ${html(durationLabel(billing.runtime.leaseDurationMs))} lease`}
${billing.assistant ? `
External assistant${html(usdLabel(billing.assistant.estimatedCostUsd))} est.${html(billing.assistant.model)} · ${billing.assistant.requests} requests · ${html(tokenLabel(billing.assistant.inputTokens + billing.assistant.cachedInputTokens))} in / ${html(tokenLabel(billing.assistant.outputTokens))} out
` : ""}
` : ""; return `
${features.length ? `
${features.map((feature) => ``, ).join("")}
` : ""} ${passed ? "" : outcome} ${!entry ? `

No result artifact was uploaded.

` : ""} ${gallery}
Test details${matcherResults.length ? `${passedMatchers}/${matcherResults.length} checks${notReached ? ` · ${notReached} not reached` : ""}` : "No checks recorded"}${entry ? ` · ${html(durationLabel(entry.result.durationMs))}` : ""} ${passed ? outcome : ""} ${billingStrip} ${links} ${renderFirstTaskDetails(entry?.result)} ${html(execution.id)} ${capabilities.length ? `
    ${capabilities.map((capability) => `
  • ${html(capability.feature.title)} · ${html(capability.label)}
  • `).join("")}
` : ""} ${ entry ? `
Attempt
${entry.result.attempt}
Duration
${html(durationLabel(entry.result.durationMs))}
Agent runtime
${html(durationLabel(billing!.runtime.agentRunDurationMs))}
${billing!.runtime.leaseDurationMs === null ? "" : `
Environment lease
${html(durationLabel(billing!.runtime.leaseDurationMs))}
`}
Runtime
${html(entry.result.runtimeMode)}
Provider
${html(entry.result.provider)}
Model
${html(entry.result.model)}
${entry.result.issueIdentifier ? `
Issue
${html(entry.result.issueIdentifier)}
` : ""}
` : "" }

${html(detail)}

${turnTimingRows ? `
${turnTimingRows}
TurnRunLeaseSchedulerRun durationFinal settlement (legacy response field)
` : ""} ${latencyRows ? `
${latencyRows}
EventTiming verdictPossible latencyTargetResolution
` : ""} ${startupReuseRows ? `
${startupReuseRows}
Startup reuse evidence
RunWarmthEnvironment/leaseWorkspaceRunnerProvider processProvider sessionProvider launches
` : ""} ${matcherRows ? `
${matcherRows}
ResultMatcherExpectationDetail
` : `

No matcher result was recorded.

`} ${entry ? `
Usage and billing metadata
${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null }, null, 2))}
` : ""}
`; } function renderSuiteResults(input: { suiteCatalog: readonly ReportExecution[]; expected: ReadonlySet; entryById: ReadonlyMap; summary?: RunnerE2ESuiteSummary; }) { const suite = input.suiteCatalog[0]?.suite; if (!suite) return ""; const summary = input.summary; const summaryHtml = summary ? `
Pass rate${summary.selected > 0 ? ((summary.passed / summary.selected) * 100).toFixed(1) : "0.0"}%${summary.passed}/${summary.selected} passed${summary.incomplete ? ` · ${summary.incomplete} incomplete` : ""}
Tokens${html(tokenLabel(summary.billing.llm.totalTokens, summary.billing.llm))}${html(tokenLabel(summary.billing.llm.inputTokens, summary.billing.llm))} input · ${html(tokenLabel(summary.billing.llm.outputTokens, summary.billing.llm))} output
Known spend${html(summary.billing.llm.runsWithReportedCost > 0 || summary.billing.estimatedRuntimeCostUsd > 0 || summary.billing.judge ? `${usdLabel(summary.billing.observedAndEstimatedCostUsd)}${summary.billing.testsWithCompleteBilling < summary.billing.testCount ? " (partial)" : ""}` : "Unavailable")}reported LLM + runtime${summary.billing.judge ? " + judge" : ""} estimate
Agent time${html(durationLabel(summary.billing.agentRunDurationMs))}${html(durationLabel(summary.billing.leaseDurationMs))} lease
Execution${summary.executed}/${summary.selected}${summary.retries} retries · cleanup ${summary.cleanupPassed ? "passed" : "failed"}
` : ""; const journeys = [...new Map(input.suiteCatalog.map(row => [row.task.id, row.task])).values()]; return `

Test suite

${html(suite.label)}

${html(suite.description)}

${summaryHtml} ${journeys.map(task => { const key = `${suite.id}.${task.id}`; const executions = input.suiteCatalog.filter(row => row.task.id === task.id); const href = `?view=case&case=${encodeURIComponent(key)}`; return `
${executions.map(row => `
${html(row.profile.label)}${html(row.environment.label)}${html(row.profile.provider)} · ${html(row.profile.model)}
${renderCase(row, input.expected, input.entryById)}
`).join("")}
`; }).join("")}
`; } export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { const expected = new Set(input.expected); const catalog = discoverReportCatalog({ catalog: input.catalog, expected: input.expected, results: input.entries.map((entry) => entry.result), }).filter((execution) => expected.has(execution.id)); const entryById = new Map( input.entries.map((entry) => [entry.result.executionId, entry]), ); const selectedEntries = input.entries.filter((entry) => expected.has(entry.result.executionId), ); const passed = selectedEntries.filter((entry) => entry.valid).length; const incomplete = selectedEntries.filter((entry) => !entry.valid && isIncompleteFirstTaskResult(entry.result, entry.errors), ).length; const failed = input.expected.length - passed - incomplete; const totalDuration = selectedEntries.reduce( (total, entry) => total + entry.result.durationMs, 0, ); const screenshotCount = selectedEntries.reduce( (total, entry) => total + resolveScreenshots(entry).length, 0, ); const campaignBilling = aggregateCampaignBilling( selectedEntries.map((entry) => entry.result), ); const suites = [ ...new Map( catalog.map((execution) => [execution.suite.id, execution.suite]), ).values(), ]; const suiteSections = suites .map((suite) => renderSuiteResults({ suiteCatalog: catalog.filter( (execution) => execution.suite.id === suite.id, ), expected, entryById, summary: input.campaign?.suites.find( (summary) => summary.suiteId === suite.id, ), }), ) .join(""); const filterProfiles = [ ...new Map( catalog.map((execution) => [execution.profile.id, execution.profile]), ).values(), ]; const filterEnvironments = [ ...new Map( catalog.map((execution) => [ execution.environment.id, execution.environment, ]), ).values(), ]; return ` ${html(input.title)} · Paperclip ${renderReportShell({ campaignId: input.campaign?.campaignId, catalog, history: input.history })}

Full-stack acceptance campaign

${html(input.title)}

Results across runner profiles and execution environments. Declared PNG screenshots and normalized results are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.

${input.campaign ? `

${html(input.generatedAt)}${input.campaign.source.sha ? `${html(input.campaign.source.sha.slice(0,8))}` : ""}${input.campaign.source.workflowRunUrl ? `Source run ↗` : ""}

` : ""}
${passed}/${input.expected.length}Passed
${failed}Failed
${incomplete ? `
${incomplete}Incomplete
` : ""}
${html(durationLabel(totalDuration))}Test time
${renderCampaignSuiteSummary(input.campaign?.suites ?? [], catalog, "", false)}
Campaign usage and spend
${html(tokenLabel(campaignBilling.llm.inputTokens, campaignBilling.llm))}Input tokens
${html(tokenLabel(campaignBilling.llm.outputTokens, campaignBilling.llm))}Output tokens
${html(tokenLabel(campaignBilling.llm.cachedInputTokens, campaignBilling.llm))}Cached tokens
${html(llmCostLabel(campaignBilling.reportedLlmCostUsd, campaignBilling.llm))}LLM reported subtotal
${html(usdLabel(campaignBilling.estimatedRuntimeCostUsd))}Daytona list estimate
${campaignBilling.judge ? `
${html(usdLabel(campaignBilling.judge.estimatedCostUsd))}Judge estimate · ${campaignBilling.judge.attempts} attempts · ${campaignBilling.judge.attemptsWithUnknownUsage} unknown usage${html(tokenLabel(campaignBilling.judge.inputTokens))} in / ${html(tokenLabel(campaignBilling.judge.outputTokens))} out · $${campaignBilling.judge.reservedCostUsd.toFixed(6)} reserved
` : ""} ${campaignBilling.assistant ? `
${html(usdLabel(campaignBilling.assistant.estimatedCostUsd))}External assistant estimate · ${campaignBilling.assistant.requests} requests${html(tokenLabel(campaignBilling.assistant.inputTokens + campaignBilling.assistant.cachedInputTokens))} in / ${html(tokenLabel(campaignBilling.assistant.outputTokens))} out
` : ""}
${html(durationLabel(campaignBilling.agentRunDurationMs))}Agent execution time
${html(durationLabel(campaignBilling.leaseDurationMs))}Daytona lease time
${campaignBilling.llm.runsWithReportedCost}/${campaignBilling.llm.runCount}Runs provider-priced

Model spend is the provider-reported subtotal; unpriced or unavailable runs are excluded, never counted as free. Daytona runtime is a public-list-price estimate from captured lease time and pinned resources, before credits, discounts, storage allowance, or invoice adjustments. Local execution has no external runtime meter.

All results

${renderJourneyFeatureCoverage(catalog, entryById)}

${suiteSections}
Generated ${html(input.generatedAt)}${catalog.length} selected executions · Declared screenshots and normalized results published
`; }