feat(runner): add eval inspector to DevTools

packages/paperclip-runner/devtools/issue-thread/src/App.tsx

@@ -24,7 +24,7 @@ import {
 } from "./live-client";
 import { parseCapabilityRoute, capabilityRouteHref, type CapabilityRoute } from "./route";
 import { SurfaceNav, type CapabilityChatHistoryItem } from "./SurfaceNav";
-import { TurnGroup, type EvalAssertion } from "./ThreadItems";
+import { EvalAssertions, TurnGroup, type EvalAssertion } from "./ThreadItems";
 
 const PANEL_OPEN_KEY = "paperclip-runner.capability.panel.open";
 /**
@@ -52,8 +52,11 @@ const CHAT_HISTORY_KEY = "paperclip-runner.capability.chat.history.v1";
 
 interface EmbeddedEvalCheck {
   id: string;
+  title: string;
+  description: string;
   passed: boolean;
   detail: string;
+  definition: Record<string, unknown>;
   anchor: { kind: "item" | "turn" | "run"; id: string };
 }
 
@@ -76,6 +79,7 @@ interface EmbeddedEvalReport {
     finalRevision: number;
   };
   view: CapabilityIssueThreadSnapshot;
+  devtools: CapabilityDevtoolsSnapshot;
 }
 
 declare global {
@@ -238,9 +242,8 @@ export function App() {
   const [actionError, setActionError] = useState<string | null>(null);
   const [loadNonce, setLoadNonce] = useState(0);
   const [settled, setSettled] = useState(false);
-  const [panelOpen, setPanelOpen] = useState(() =>
-    readStoredFlag(route.surface === "chat" ? CHAT_PANEL_OPEN_KEY : PANEL_OPEN_KEY, false),
-  );
+  const [panelOpen, setPanelOpen] = useState(() => embeddedEval !== null ||
+    readStoredFlag(route.surface === "chat" ? CHAT_PANEL_OPEN_KEY : PANEL_OPEN_KEY, false));
   const [panelWidth, setPanelWidth] = useState(() => readStoredNumber(PANEL_WIDTH_KEY, 384));
   const [segment, setSegment] = useState<"thread" | "evidence">(route.segment);
   const [openSections, setOpenSections] = useState<CapabilityEvidenceSectionId[]>(["tools"]);
@@ -287,6 +290,7 @@ export function App() {
         ...embeddedEval.view,
         composer: { state: "disabled", helper: null, reason: "Immutable eval recording", pendingInteractionId: null },
       });
+      setDevtools(embeddedEval.devtools);
       setSnapshotSurface("issue");
     } else if (chat) {
       // The clean room has no fixture fallback: if real Codex and real runnerd
@@ -871,30 +875,6 @@ export function App() {
         onSelectSegment={setSegment}
       />
 
-      {embeddedEval !== null ? (
-        <section className="pit-eval-summary" data-passed={embeddedEval.passed} data-testid="eval-summary">
-          <div className="pit-eval-summary-head">
-            <a href="../../index.html">← Eval suite</a>
-            <strong>{embeddedEval.passed ? "PASS" : embeddedEval.disposition.replaceAll("_", " ").toUpperCase()}</strong>
-            <code>{embeddedEval.attemptId}</code>
-          </div>
-          <dl className="pit-eval-run-facts">
-            <div><dt>Model</dt><dd>{embeddedEval.run.provider}/{embeddedEval.run.model}</dd></div>
-            <div><dt>Session</dt><dd>{embeddedEval.run.sessionId}</dd></div>
-            <div><dt>Duration</dt><dd>{Math.max(0, new Date(embeddedEval.run.finishedAt).getTime() - new Date(embeddedEval.run.startedAt).getTime())} ms</dd></div>
-            <div><dt>Runner</dt><dd>{embeddedEval.run.runnerPackageDigest.slice(0, 20)}…</dd></div>
-            <div><dt>runnerd</dt><dd>{embeddedEval.run.runnerdDigest.slice(0, 20)}…</dd></div>
-          </dl>
-          <div className="pit-eval-checks">
-            {embeddedEval.checks.map((check) => (
-              <span key={check.id} data-passed={check.passed} title={check.detail}>
-                {check.passed ? "✓" : "✕"} {check.id}
-              </span>
-            ))}
-          </div>
-        </section>
-      ) : null}
-
       {actionError !== null ? (
         <p className="pit-banner" data-tone="danger" role="alert" data-testid="action-error">
           <span aria-hidden="true">⚠</span>
@@ -996,16 +976,7 @@ export function App() {
             >
               <div className="pit-thread">
                 {embeddedEval !== null ? (
-                  <>
-                    <div className="pit-eval-boundary" data-phase="fixture">
-                      <strong>Fixture state</strong>
-                      <span>{embeddedEval.run.fixtureDigest} · revision {embeddedEval.run.initialRevision}</span>
-                    </div>
-                    <div className="pit-eval-boundary" data-phase="execution">
-                      <strong>Eval execution begins</strong>
-                      <span>Everything below this line was produced or observed during the run.</span>
-                    </div>
-                  </>
+                  <div className="pit-eval-boundary" data-phase="execution"><strong>Eval execution</strong></div>
                 ) : null}
                 {snapshot.turns.length === 0 ? (
                   <section className="pit-empty-thread" data-testid="clean-room-empty">
@@ -1036,10 +1007,10 @@ export function App() {
                       }}
                       assertions={Object.fromEntries(embeddedEval?.checks
                         .filter((check) => check.anchor.kind === "item")
-                        .map((check) => [check.anchor.id, [{ id: check.id, passed: check.passed, detail: check.detail } satisfies EvalAssertion]]) ?? [])}
+                        .map((check) => [check.anchor.id, [{ ...check } satisfies EvalAssertion]]) ?? [])}
                       terminalAssertions={embeddedEval?.checks
                         .filter((check) => check.anchor.kind === "turn" && check.anchor.id === turn.id)
-                        .map((check) => ({ id: check.id, passed: check.passed, detail: check.detail })) ?? []}
+                        .map((check) => ({ ...check })) ?? []}
                     />
                   ))
                 )}
@@ -1047,14 +1018,7 @@ export function App() {
                   <div className="pit-eval-boundary" data-phase="post-run">
                     <strong>Post-run state</strong>
                     <span>Final mock control-plane revision {embeddedEval.run.finalRevision}</span>
-                    <div className="pit-inline-assertions">
-                      {embeddedEval.checks.filter((check) => check.anchor.kind === "run").map((check) => (
-                        <div key={check.id} data-passed={check.passed}>
-                          <strong>{check.passed ? "✓ PASS" : "✕ FAIL"} · {check.id}</strong>
-                          <span>{check.detail}</span>
-                        </div>
-                      ))}
-                    </div>
+                    <EvalAssertions assertions={embeddedEval.checks.filter((check) => check.anchor.kind === "run")} />
                   </div>
                 ) : null}
               </div>
@@ -1145,6 +1109,7 @@ export function App() {
         {showPanel ? (
           <EvidencePanel
             snapshot={snapshot}
+            evalReport={embeddedEval}
             {...(route.mode === "live" && historicSessionId === null ? { devtools } : {})}
             onForkRevision={(revision) => {
               abandonTurn();

packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx

@@ -2,10 +2,20 @@ import { useEffect, useMemo, useState } from "react";
 
 import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools";
 import { Icon, type IconName } from "./Icons";
+import { EvalAssertions, type EvalAssertion } from "./ThreadItems";
 
-export type CapabilityDevtoolsTab = "evidence" | "timeline" | "state" | "diff" | "documents" | "protocol" | "runtime" | "authority";
+export type CapabilityDevtoolsTab = "eval" | "evidence" | "timeline" | "state" | "diff" | "documents" | "protocol" | "runtime" | "authority";
 type Json = null | boolean | number | string | Json[] | { [key: string]: Json };
 
+export interface EvalInspectorReport {
+  attemptId: string;
+  caseId: string;
+  disposition: string;
+  passed: boolean;
+  checks: EvalAssertion[];
+  run: { model: string; provider: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; startedAt: string; finishedAt: string; initialRevision: number; finalRevision: number };
+}
+
 function JsonTree({ value, name = "root" }: { value: Json; name?: string }) {
   if (value === null || typeof value !== "object") {
     return <span className="pit-json-value">{JSON.stringify(value)}</span>;
@@ -86,7 +96,7 @@ function documentsOf(state: Json): Array<{ id: string; key: string; title: strin
   });
 }
 
-export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange }: { snapshot: CapabilityDevtoolsSnapshot; onFork: (revision: number) => void; tab: CapabilityDevtoolsTab; onTabChange: (tab: CapabilityDevtoolsTab) => void }) {
+export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange, evalReport }: { snapshot: CapabilityDevtoolsSnapshot; onFork: (revision: number) => void; tab: CapabilityDevtoolsTab; onTabChange: (tab: CapabilityDevtoolsTab) => void; evalReport?: EvalInspectorReport | null }) {
   const [query, setQuery] = useState("");
   const latest = snapshot.revisions.at(-1)!;
   const [revision, setRevision] = useState(latest.revision);
@@ -124,10 +134,25 @@ export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange }: { snap
         <button className="pit-button" type="button" onClick={() => onFork(revision)}><Icon name="branch" /> Fork r{revision}</button>
       </div>
       <div className="pit-devtools-tabs" role="tablist" aria-label="Developer tools">
-        {TABS.map(({ id, icon, label }) => (
+        {([...(evalReport ? [{ id: "eval" as const, icon: "evidence" as const, label: "Eval" }] : []), ...TABS]).map(({ id, icon, label }) => (
           <button key={id} type="button" role="tab" aria-selected={tab === id} className="pit-tab" onClick={() => onTabChange(id)}><span className="pit-tab-glyph"><Icon name={icon} /></span><span>{label}</span></button>
         ))}
       </div>
+      {tab === "eval" && evalReport ? (
+        <div className="pit-devtools-pane pit-eval-inspector">
+          <div className="pit-eval-summary-head"><a href="../../index.html">← Eval suite</a><strong>{evalReport.passed ? "PASS" : evalReport.disposition.replaceAll("_", " ").toUpperCase()}</strong><code>{evalReport.attemptId}</code></div>
+          <dl className="pit-eval-run-facts">
+            <div><dt>Model</dt><dd>{evalReport.run.provider}/{evalReport.run.model}</dd></div>
+            <div><dt>Session</dt><dd>{evalReport.run.sessionId}</dd></div>
+            <div><dt>Duration</dt><dd>{Math.max(0, new Date(evalReport.run.finishedAt).getTime() - new Date(evalReport.run.startedAt).getTime())} ms</dd></div>
+            <div><dt>Fixture</dt><dd>{evalReport.run.fixtureDigest}</dd></div>
+            <div><dt>State</dt><dd>r{evalReport.run.initialRevision} → r{evalReport.run.finalRevision}</dd></div>
+            <div><dt>Runner</dt><dd>{evalReport.run.runnerPackageDigest}</dd></div>
+            <div><dt>runnerd</dt><dd>{evalReport.run.runnerdDigest}</dd></div>
+          </dl>
+          <h3>Assertions</h3><EvalAssertions assertions={evalReport.checks} />
+        </div>
+      ) : null}
       {tab === "timeline" ? (
         <div className="pit-devtools-list">
           {revisions.map((entry) => (

packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.tsx

@@ -10,7 +10,7 @@ import type {
   CapabilityToolDisposition,
 } from "../../../src/issue-thread/types";
 import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools";
-import { DevtoolsInspector, type CapabilityDevtoolsTab } from "./DevtoolsInspector";
+import { DevtoolsInspector, type CapabilityDevtoolsTab, type EvalInspectorReport } from "./DevtoolsInspector";
 import { Icon } from "./Icons";
 import { capabilitySemanticToolDescriptor } from "../../../src/semantic-tools/catalog";
 import {
@@ -113,6 +113,7 @@ function RunnerEventGroup({ group }: { group: CapabilityRunnerEventGroup }) {
 export interface EvidencePanelProps {
   snapshot: CapabilityIssueThreadSnapshot;
   devtools?: CapabilityDevtoolsSnapshot | null;
+  evalReport?: EvalInspectorReport | null;
   onForkRevision?: (revision: number) => void;
   layout: "side" | "overlay" | "segment";
   width: number;
@@ -330,10 +331,11 @@ function Section({
 }
 
 export function EvidencePanel(props: EvidencePanelProps) {
-  const [devtoolsTab, setDevtoolsTab] = useState<CapabilityDevtoolsTab>("evidence");
+  const [devtoolsTab, setDevtoolsTab] = useState<CapabilityDevtoolsTab>(props.evalReport ? "eval" : "evidence");
   const {
     snapshot,
     devtools,
+    evalReport,
     onForkRevision = () => undefined,
     layout,
     width,
@@ -392,7 +394,7 @@ export function EvidencePanel(props: EvidencePanelProps) {
           {devtools === null ? (
             <p className="pit-muted pit-devtools-loading">Loading company state…</p>
           ) : (
-            <DevtoolsInspector snapshot={devtools} onFork={onForkRevision} tab={devtoolsTab} onTabChange={setDevtoolsTab} />
+            <DevtoolsInspector snapshot={devtools} onFork={onForkRevision} tab={devtoolsTab} onTabChange={setDevtoolsTab} evalReport={evalReport} />
           )}
         </>
       ) : null}

packages/paperclip-runner/devtools/issue-thread/src/ThreadItems.tsx

@@ -12,21 +12,37 @@ import { Chip, StatusBadge, Timestamp, formatBytes } from "./primitives";
 
 export interface EvalAssertion {
   id: string;
+  title: string;
+  description: string;
   passed: boolean;
   detail: string;
+  definition: Record<string, unknown>;
 }
 
-function EvalAssertions({ assertions }: { assertions: EvalAssertion[] }) {
+export function EvalAssertions({ assertions }: { assertions: EvalAssertion[] }) {
+  const [selected, setSelected] = useState<EvalAssertion | null>(null);
   if (assertions.length === 0) return null;
   return (
-    <div className="pit-inline-assertions" aria-label="Eval assertions">
-      {assertions.map((assertion) => (
-        <div key={assertion.id} data-passed={assertion.passed}>
-          <strong>{assertion.passed ? "✓ PASS" : "✕ FAIL"} · {assertion.id}</strong>
-          <span>{assertion.detail}</span>
+    <>
+      <div className="pit-inline-assertions" aria-label="Eval assertions">
+        {assertions.map((assertion) => (
+          <button type="button" key={assertion.id} data-passed={assertion.passed} onClick={() => setSelected(assertion)}>
+            <strong>{assertion.passed ? "✓ PASS" : "✕ FAIL"} · {assertion.title}</strong>
+            <span>{assertion.detail}</span>
+          </button>
+        ))}
+      </div>
+      {selected !== null ? (
+        <div className="pit-dialog-backdrop" onMouseDown={(event) => { if (event.target === event.currentTarget) setSelected(null); }}>
+          <div className="pit-dialog pit-eval-check-dialog" role="dialog" aria-modal="true" aria-labelledby="eval-check-title">
+            <header className="pit-tool-dialog-head"><div><span className="pit-card-meta">Eval assertion · {selected.id}</span><h2 id="eval-check-title">{selected.title}</h2></div><button type="button" className="pit-icon-button" aria-label="Close assertion" onClick={() => setSelected(null)}>×</button></header>
+            <p>{selected.description}</p>
+            <p className="pit-card-meta">Observed result</p><p>{selected.detail}</p>
+            <p className="pit-card-meta">Original case definition</p><pre className="pit-code">{JSON.stringify(selected.definition, null, 2)}</pre>
+          </div>
         </div>
-      ))}
-    </div>
+      ) : null}
+    </>
   );
 }
 

packages/paperclip-runner/devtools/issue-thread/src/issue-thread.css

@@ -1266,16 +1266,21 @@ textarea {
 .pit-eval-run-facts dt { color: var(--pit-muted-foreground); font-size: var(--pit-text-xs); }
 .pit-eval-run-facts dd { margin: 2px 0 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-family: var(--pit-font-mono); font-size: var(--pit-text-xs); }
 .pit-eval-boundary { display: grid; gap: 3px; margin: 14px 0; padding: 10px 12px; border-block: 1px solid var(--pit-border-strong); background: var(--pit-surface-sunken); }
-.pit-eval-boundary[data-phase="execution"] { border-color: var(--pit-live); }
+.pit-eval-boundary[data-phase="execution"] { display: flex; margin: 4px 0; padding: 7px 0; border: 0; border-top: 2px solid var(--pit-live); background: transparent; color: var(--pit-live); }
 .pit-eval-boundary[data-phase="post-run"] { border-color: var(--pit-accent); }
 .pit-eval-boundary span { color: var(--pit-muted-foreground); font-family: var(--pit-font-mono); font-size: var(--pit-text-xs); overflow-wrap: anywhere; }
 .pit-eval-item { display: contents; }
 .pit-inline-assertions { display: grid; gap: 6px; margin: 6px 0 12px; }
-.pit-inline-assertions > div { display: flex; align-items: baseline; gap: 10px; padding: 7px 10px; border-left: 3px solid var(--pit-danger); background: var(--pit-danger-surface); font-size: var(--pit-text-xs); }
-.pit-inline-assertions > div[data-passed="true"] { border-left-color: var(--pit-success); background: var(--pit-success-surface); }
-.pit-inline-assertions > div strong { color: var(--pit-danger); white-space: nowrap; }
-.pit-inline-assertions > div[data-passed="true"] strong { color: var(--pit-success); }
-.pit-inline-assertions > div span { color: var(--pit-muted-foreground); }
+.pit-inline-assertions > button { display: flex; align-items: baseline; gap: 10px; width: 100%; padding: 7px 10px; border: 0; border-left: 3px solid var(--pit-danger); background: var(--pit-danger-surface); color: inherit; text-align: left; cursor: pointer; font-size: var(--pit-text-xs); }
+.pit-inline-assertions > button[data-passed="true"] { border-left-color: var(--pit-success); background: var(--pit-success-surface); }
+.pit-inline-assertions > button strong { color: var(--pit-danger); white-space: nowrap; }
+.pit-inline-assertions > button[data-passed="true"] strong { color: var(--pit-success); }
+.pit-inline-assertions > button span { color: var(--pit-muted-foreground); }
+.pit-inline-assertions > button:hover { filter: brightness(1.12); }
+.pit-eval-inspector .pit-eval-run-facts { grid-template-columns: minmax(0, 1fr); }
+.pit-eval-inspector .pit-eval-run-facts dd { white-space: normal; overflow-wrap: anywhere; }
+.pit-eval-check-dialog { width: min(42rem, 100%); }
+.pit-eval-check-dialog .pit-code { max-height: 22rem; overflow: auto; }
 @media(max-width:900px){.pit-eval-run-facts{grid-template-columns:repeat(2,minmax(0,1fr))}}
 
 @keyframes pit-slide {