From 5316e6ded61758dbd4b1699517980f86271610cb Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 20 Aug 2026 15:03:39 -0500 Subject: [PATCH] feat(runner): add eval inspector to DevTools --- .../devtools/issue-thread/src/App.tsx | 61 ++++--------------- .../issue-thread/src/DevtoolsInspector.tsx | 31 +++++++++- .../issue-thread/src/EvidencePanel.tsx | 8 ++- .../devtools/issue-thread/src/ThreadItems.tsx | 32 +++++++--- .../issue-thread/src/issue-thread.css | 17 ++++-- 5 files changed, 81 insertions(+), 68 deletions(-) diff --git a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx index b9ca12ebe48..67470df8473 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx @@ -24,7 +24,7 @@ import { } from "./live-client"; import { parseCapabilityRoute, capabilityRouteHref, type CapabilityRoute } from "./route"; import { SurfaceNav, type CapabilityChatHistoryItem } from "./SurfaceNav"; -import { TurnGroup, type EvalAssertion } from "./ThreadItems"; +import { EvalAssertions, TurnGroup, type EvalAssertion } from "./ThreadItems"; const PANEL_OPEN_KEY = "paperclip-runner.capability.panel.open"; /** @@ -52,8 +52,11 @@ const CHAT_HISTORY_KEY = "paperclip-runner.capability.chat.history.v1"; interface EmbeddedEvalCheck { id: string; + title: string; + description: string; passed: boolean; detail: string; + definition: Record; anchor: { kind: "item" | "turn" | "run"; id: string }; } @@ -76,6 +79,7 @@ interface EmbeddedEvalReport { finalRevision: number; }; view: CapabilityIssueThreadSnapshot; + devtools: CapabilityDevtoolsSnapshot; } declare global { @@ -238,9 +242,8 @@ export function App() { const [actionError, setActionError] = useState(null); const [loadNonce, setLoadNonce] = useState(0); const [settled, setSettled] = useState(false); - const [panelOpen, setPanelOpen] = useState(() => - readStoredFlag(route.surface === "chat" ? CHAT_PANEL_OPEN_KEY : PANEL_OPEN_KEY, false), - ); + const [panelOpen, setPanelOpen] = useState(() => embeddedEval !== null || + readStoredFlag(route.surface === "chat" ? CHAT_PANEL_OPEN_KEY : PANEL_OPEN_KEY, false)); const [panelWidth, setPanelWidth] = useState(() => readStoredNumber(PANEL_WIDTH_KEY, 384)); const [segment, setSegment] = useState<"thread" | "evidence">(route.segment); const [openSections, setOpenSections] = useState(["tools"]); @@ -287,6 +290,7 @@ export function App() { ...embeddedEval.view, composer: { state: "disabled", helper: null, reason: "Immutable eval recording", pendingInteractionId: null }, }); + setDevtools(embeddedEval.devtools); setSnapshotSurface("issue"); } else if (chat) { // The clean room has no fixture fallback: if real Codex and real runnerd @@ -871,30 +875,6 @@ export function App() { onSelectSegment={setSegment} /> - {embeddedEval !== null ? ( -
-
- ← Eval suite - {embeddedEval.passed ? "PASS" : embeddedEval.disposition.replaceAll("_", " ").toUpperCase()} - {embeddedEval.attemptId} -
-
-
Model
{embeddedEval.run.provider}/{embeddedEval.run.model}
-
Session
{embeddedEval.run.sessionId}
-
Duration
{Math.max(0, new Date(embeddedEval.run.finishedAt).getTime() - new Date(embeddedEval.run.startedAt).getTime())} ms
-
Runner
{embeddedEval.run.runnerPackageDigest.slice(0, 20)}…
-
runnerd
{embeddedEval.run.runnerdDigest.slice(0, 20)}…
-
-
- {embeddedEval.checks.map((check) => ( - - {check.passed ? "✓" : "✕"} {check.id} - - ))} -
-
- ) : null} - {actionError !== null ? (

@@ -996,16 +976,7 @@ export function App() { >

{embeddedEval !== null ? ( - <> -
- Fixture state - {embeddedEval.run.fixtureDigest} · revision {embeddedEval.run.initialRevision} -
-
- Eval execution begins - Everything below this line was produced or observed during the run. -
- +
Eval execution
) : null} {snapshot.turns.length === 0 ? (
@@ -1036,10 +1007,10 @@ export function App() { }} assertions={Object.fromEntries(embeddedEval?.checks .filter((check) => check.anchor.kind === "item") - .map((check) => [check.anchor.id, [{ id: check.id, passed: check.passed, detail: check.detail } satisfies EvalAssertion]]) ?? [])} + .map((check) => [check.anchor.id, [{ ...check } satisfies EvalAssertion]]) ?? [])} terminalAssertions={embeddedEval?.checks .filter((check) => check.anchor.kind === "turn" && check.anchor.id === turn.id) - .map((check) => ({ id: check.id, passed: check.passed, detail: check.detail })) ?? []} + .map((check) => ({ ...check })) ?? []} /> )) )} @@ -1047,14 +1018,7 @@ export function App() {
Post-run state Final mock control-plane revision {embeddedEval.run.finalRevision} -
- {embeddedEval.checks.filter((check) => check.anchor.kind === "run").map((check) => ( -
- {check.passed ? "✓ PASS" : "✕ FAIL"} · {check.id} - {check.detail} -
- ))} -
+ check.anchor.kind === "run")} />
) : null}
@@ -1145,6 +1109,7 @@ export function App() { {showPanel ? ( { abandonTurn(); diff --git a/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx b/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx index 45e64ce73b7..d795d94a3e2 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx @@ -2,10 +2,20 @@ import { useEffect, useMemo, useState } from "react"; import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools"; import { Icon, type IconName } from "./Icons"; +import { EvalAssertions, type EvalAssertion } from "./ThreadItems"; -export type CapabilityDevtoolsTab = "evidence" | "timeline" | "state" | "diff" | "documents" | "protocol" | "runtime" | "authority"; +export type CapabilityDevtoolsTab = "eval" | "evidence" | "timeline" | "state" | "diff" | "documents" | "protocol" | "runtime" | "authority"; type Json = null | boolean | number | string | Json[] | { [key: string]: Json }; +export interface EvalInspectorReport { + attemptId: string; + caseId: string; + disposition: string; + passed: boolean; + checks: EvalAssertion[]; + run: { model: string; provider: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; startedAt: string; finishedAt: string; initialRevision: number; finalRevision: number }; +} + function JsonTree({ value, name = "root" }: { value: Json; name?: string }) { if (value === null || typeof value !== "object") { return {JSON.stringify(value)}; @@ -86,7 +96,7 @@ function documentsOf(state: Json): Array<{ id: string; key: string; title: strin }); } -export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange }: { snapshot: CapabilityDevtoolsSnapshot; onFork: (revision: number) => void; tab: CapabilityDevtoolsTab; onTabChange: (tab: CapabilityDevtoolsTab) => void }) { +export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange, evalReport }: { snapshot: CapabilityDevtoolsSnapshot; onFork: (revision: number) => void; tab: CapabilityDevtoolsTab; onTabChange: (tab: CapabilityDevtoolsTab) => void; evalReport?: EvalInspectorReport | null }) { const [query, setQuery] = useState(""); const latest = snapshot.revisions.at(-1)!; const [revision, setRevision] = useState(latest.revision); @@ -124,10 +134,25 @@ export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange }: { snap
- {TABS.map(({ id, icon, label }) => ( + {([...(evalReport ? [{ id: "eval" as const, icon: "evidence" as const, label: "Eval" }] : []), ...TABS]).map(({ id, icon, label }) => ( ))}
+ {tab === "eval" && evalReport ? ( +
+
← Eval suite{evalReport.passed ? "PASS" : evalReport.disposition.replaceAll("_", " ").toUpperCase()}{evalReport.attemptId}
+
+
Model
{evalReport.run.provider}/{evalReport.run.model}
+
Session
{evalReport.run.sessionId}
+
Duration
{Math.max(0, new Date(evalReport.run.finishedAt).getTime() - new Date(evalReport.run.startedAt).getTime())} ms
+
Fixture
{evalReport.run.fixtureDigest}
+
State
r{evalReport.run.initialRevision} → r{evalReport.run.finalRevision}
+
Runner
{evalReport.run.runnerPackageDigest}
+
runnerd
{evalReport.run.runnerdDigest}
+
+

Assertions

+
+ ) : null} {tab === "timeline" ? (
{revisions.map((entry) => ( diff --git a/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx b/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx index 6c3ca04ae45..61aa2d73735 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx @@ -10,7 +10,7 @@ import type { CapabilityToolDisposition, } from "../../../src/issue-thread/types"; import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools"; -import { DevtoolsInspector, type CapabilityDevtoolsTab } from "./DevtoolsInspector"; +import { DevtoolsInspector, type CapabilityDevtoolsTab, type EvalInspectorReport } from "./DevtoolsInspector"; import { Icon } from "./Icons"; import { capabilitySemanticToolDescriptor } from "../../../src/semantic-tools/catalog"; import { @@ -113,6 +113,7 @@ function RunnerEventGroup({ group }: { group: CapabilityRunnerEventGroup }) { export interface EvidencePanelProps { snapshot: CapabilityIssueThreadSnapshot; devtools?: CapabilityDevtoolsSnapshot | null; + evalReport?: EvalInspectorReport | null; onForkRevision?: (revision: number) => void; layout: "side" | "overlay" | "segment"; width: number; @@ -330,10 +331,11 @@ function Section({ } export function EvidencePanel(props: EvidencePanelProps) { - const [devtoolsTab, setDevtoolsTab] = useState("evidence"); + const [devtoolsTab, setDevtoolsTab] = useState(props.evalReport ? "eval" : "evidence"); const { snapshot, devtools, + evalReport, onForkRevision = () => undefined, layout, width, @@ -392,7 +394,7 @@ export function EvidencePanel(props: EvidencePanelProps) { {devtools === null ? (

Loading company state…

) : ( - + )} ) : null} diff --git a/packages/paperclip-runner/devtools/issue-thread/src/ThreadItems.tsx b/packages/paperclip-runner/devtools/issue-thread/src/ThreadItems.tsx index 4feb4b5c0fa..70a68af7a38 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/ThreadItems.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/ThreadItems.tsx @@ -12,21 +12,37 @@ import { Chip, StatusBadge, Timestamp, formatBytes } from "./primitives"; export interface EvalAssertion { id: string; + title: string; + description: string; passed: boolean; detail: string; + definition: Record; } -function EvalAssertions({ assertions }: { assertions: EvalAssertion[] }) { +export function EvalAssertions({ assertions }: { assertions: EvalAssertion[] }) { + const [selected, setSelected] = useState(null); if (assertions.length === 0) return null; return ( -
- {assertions.map((assertion) => ( -
- {assertion.passed ? "✓ PASS" : "✕ FAIL"} · {assertion.id} - {assertion.detail} + <> +
+ {assertions.map((assertion) => ( + + ))} +
+ {selected !== null ? ( +
{ if (event.target === event.currentTarget) setSelected(null); }}> +
+
Eval assertion · {selected.id}

{selected.title}

+

{selected.description}

+

Observed result

{selected.detail}

+

Original case definition

{JSON.stringify(selected.definition, null, 2)}
+
- ))} -
+ ) : null} + ); } diff --git a/packages/paperclip-runner/devtools/issue-thread/src/issue-thread.css b/packages/paperclip-runner/devtools/issue-thread/src/issue-thread.css index efa9e3fe5ac..201544e428c 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/issue-thread.css +++ b/packages/paperclip-runner/devtools/issue-thread/src/issue-thread.css @@ -1266,16 +1266,21 @@ textarea { .pit-eval-run-facts dt { color: var(--pit-muted-foreground); font-size: var(--pit-text-xs); } .pit-eval-run-facts dd { margin: 2px 0 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-family: var(--pit-font-mono); font-size: var(--pit-text-xs); } .pit-eval-boundary { display: grid; gap: 3px; margin: 14px 0; padding: 10px 12px; border-block: 1px solid var(--pit-border-strong); background: var(--pit-surface-sunken); } -.pit-eval-boundary[data-phase="execution"] { border-color: var(--pit-live); } +.pit-eval-boundary[data-phase="execution"] { display: flex; margin: 4px 0; padding: 7px 0; border: 0; border-top: 2px solid var(--pit-live); background: transparent; color: var(--pit-live); } .pit-eval-boundary[data-phase="post-run"] { border-color: var(--pit-accent); } .pit-eval-boundary span { color: var(--pit-muted-foreground); font-family: var(--pit-font-mono); font-size: var(--pit-text-xs); overflow-wrap: anywhere; } .pit-eval-item { display: contents; } .pit-inline-assertions { display: grid; gap: 6px; margin: 6px 0 12px; } -.pit-inline-assertions > div { display: flex; align-items: baseline; gap: 10px; padding: 7px 10px; border-left: 3px solid var(--pit-danger); background: var(--pit-danger-surface); font-size: var(--pit-text-xs); } -.pit-inline-assertions > div[data-passed="true"] { border-left-color: var(--pit-success); background: var(--pit-success-surface); } -.pit-inline-assertions > div strong { color: var(--pit-danger); white-space: nowrap; } -.pit-inline-assertions > div[data-passed="true"] strong { color: var(--pit-success); } -.pit-inline-assertions > div span { color: var(--pit-muted-foreground); } +.pit-inline-assertions > button { display: flex; align-items: baseline; gap: 10px; width: 100%; padding: 7px 10px; border: 0; border-left: 3px solid var(--pit-danger); background: var(--pit-danger-surface); color: inherit; text-align: left; cursor: pointer; font-size: var(--pit-text-xs); } +.pit-inline-assertions > button[data-passed="true"] { border-left-color: var(--pit-success); background: var(--pit-success-surface); } +.pit-inline-assertions > button strong { color: var(--pit-danger); white-space: nowrap; } +.pit-inline-assertions > button[data-passed="true"] strong { color: var(--pit-success); } +.pit-inline-assertions > button span { color: var(--pit-muted-foreground); } +.pit-inline-assertions > button:hover { filter: brightness(1.12); } +.pit-eval-inspector .pit-eval-run-facts { grid-template-columns: minmax(0, 1fr); } +.pit-eval-inspector .pit-eval-run-facts dd { white-space: normal; overflow-wrap: anywhere; } +.pit-eval-check-dialog { width: min(42rem, 100%); } +.pit-eval-check-dialog .pit-code { max-height: 22rem; overflow: auto; } @media(max-width:900px){.pit-eval-run-facts{grid-template-columns:repeat(2,minmax(0,1fr))}} @keyframes pit-slide {