import { useEffect, useMemo, useState } from "react"; import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools"; import { Icon, type IconName } from "./Icons"; import { EvalAssertions, type EvalAssertion } from "./ThreadItems"; export type CapabilityDevtoolsTab = "eval" | "evidence" | "timeline" | "state" | "diff" | "documents" | "protocol" | "runtime" | "authority"; type Json = null | boolean | number | string | Json[] | { [key: string]: Json }; export interface EvalInspectorReport { attemptId: string; caseId: string; disposition: string; passed: boolean; checks: EvalAssertion[]; run: { model: string; provider: string; configuration: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; runnerBuild: string; startedAt: string; finishedAt: string; durationMs: number; initialRevision: number; finalRevision: number; usage: { agentTurns: number; providerRequests: number | null; inputTokens: number; outputTokens: number; cachedInputTokens: number; reasoningTokens: number; estimatedCostNanodollars: number; pricingVersion: string } | null }; } function JsonTree({ value, name = "root" }: { value: Json; name?: string }) { if (value === null || typeof value !== "object") { return {JSON.stringify(value)}; } const entries = Array.isArray(value) ? value.map((entry, index) => [String(index), entry] as const) : Object.entries(value); return (
{name} {Array.isArray(value) ? `[${entries.length}]` : `{${entries.length}}`}
{entries.map(([key, entry]) => (
{key} {entry !== null && typeof entry === "object" ? : }
))}
); } interface DiffRow { path: string; before: Json | undefined; after: Json | undefined } function diff(before: Json | undefined, after: Json | undefined, path = "state"): DiffRow[] { if (JSON.stringify(before) === JSON.stringify(after)) return []; if ( before !== null && after !== null && typeof before === "object" && typeof after === "object" && !Array.isArray(before) && !Array.isArray(after) ) { const keys = new Set([...Object.keys(before), ...Object.keys(after)]); return [...keys].flatMap((key) => diff(before[key], after[key], `${path}.${key}`)); } return [{ path, before, after }]; } function download(snapshot: CapabilityDevtoolsSnapshot) { const blob = new Blob([JSON.stringify(snapshot, null, 2)], { type: "application/json" }); const url = URL.createObjectURL(blob); const anchor = document.createElement("a"); anchor.href = url; anchor.download = `paperclip-devtools-r${snapshot.currentRevision}.json`; anchor.click(); URL.revokeObjectURL(url); } const TABS: ReadonlyArray<{ id: CapabilityDevtoolsTab; icon: IconName; label: string }> = [ { id: "evidence", icon: "evidence", label: "Evidence" }, { id: "timeline", icon: "timeline", label: "Timeline" }, { id: "state", icon: "state", label: "State" }, { id: "diff", icon: "diff", label: "Diff" }, { id: "documents", icon: "documents", label: "Documents" }, { id: "protocol", icon: "protocol", label: "Protocol" }, { id: "runtime", icon: "terminal", label: "Runtime" }, { id: "authority", icon: "authority", label: "Authority" }, ]; function documentsOf(state: Json): Array<{ id: string; key: string; title: string; revisions: Array<{ id: string; revision: number; body: string; changeSummary: string | null; createdAt: string }> }> { if (state === null || typeof state !== "object" || Array.isArray(state)) return []; const documents = state.documents; if (!Array.isArray(documents)) return []; return documents.flatMap((value) => { if (value === null || typeof value !== "object" || Array.isArray(value)) return []; const revisions = Array.isArray(value.revisions) ? value.revisions.flatMap((candidate) => { if (candidate === null || typeof candidate !== "object" || Array.isArray(candidate)) return []; return [{ id: String(candidate.id ?? ""), revision: Number(candidate.revision ?? 0), body: String(candidate.body ?? ""), changeSummary: candidate.changeSummary === null ? null : String(candidate.changeSummary ?? ""), createdAt: String(candidate.createdAt ?? ""), }]; }) : []; return [{ id: String(value.id ?? ""), key: String(value.key ?? ""), title: String(value.title ?? "Untitled"), revisions }]; }); } export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange, evalReport }: { snapshot: CapabilityDevtoolsSnapshot; onFork: (revision: number) => void; tab: CapabilityDevtoolsTab; onTabChange: (tab: CapabilityDevtoolsTab) => void; evalReport?: EvalInspectorReport | null }) { const [query, setQuery] = useState(""); const latest = snapshot.revisions.at(-1)!; const [revision, setRevision] = useState(latest.revision); const [following, setFollowing] = useState(true); const [compareRevision, setCompareRevision] = useState(snapshot.revisions[0]!.revision); const selected = snapshot.revisions.find((entry) => entry.revision === revision) ?? latest; const compared = snapshot.revisions.find((entry) => entry.revision === compareRevision) ?? snapshot.revisions[0]!; const rows = useMemo( () => diff(compared.state as Json, selected.state as Json), [compared.state, selected.state], ); const normalized = query.trim().toLowerCase(); const revisions = snapshot.revisions.filter((entry) => normalized.length === 0 || JSON.stringify(entry).toLowerCase().includes(normalized), ); const protocol = snapshot.protocol.filter((entry) => normalized.length === 0 || JSON.stringify(entry).toLowerCase().includes(normalized), ); const documents = useMemo(() => documentsOf(selected.state as Json), [selected.state]); const [documentId, setDocumentId] = useState(""); const document = documents.find((entry) => entry.id === documentId) ?? documents[0] ?? null; const [documentRevision, setDocumentRevision] = useState(0); const visibleDocumentRevision = document?.revisions.find((entry) => entry.revision === documentRevision) ?? document?.revisions.at(-1) ?? null; useEffect(() => { if (following) setRevision(latest.revision); }, [following, latest.revision]); return (
State monitor setQuery(event.target.value)} placeholder="operation, entity, turn…" />
{([...(evalReport ? [{ id: "eval" as const, icon: "evidence" as const, label: "Eval" }] : []), ...TABS]).map(({ id, icon, label }) => ( ))}
{tab === "eval" && evalReport ? (
← Eval suite{evalReport.passed ? "PASS" : evalReport.disposition.replaceAll("_", " ").toUpperCase()}{evalReport.attemptId}
Model
{evalReport.run.provider}/{evalReport.run.model}
Configuration
{evalReport.run.configuration}
Session
{evalReport.run.sessionId}
Duration
{evalReport.run.durationMs} ms
Fixture
{evalReport.run.fixtureDigest}
State
r{evalReport.run.initialRevision} → r{evalReport.run.finalRevision}
Tokens
{evalReport.run.usage === null ? "unknown" : `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`}
Agent turns
{evalReport.run.usage?.agentTurns ?? "unknown"}
Provider requests
{evalReport.run.usage?.providerRequests ?? "unavailable"}
Estimated cost
{evalReport.run.usage === null ? "unknown" : `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)} · ${evalReport.run.usage.pricingVersion}`}
Runner
{evalReport.run.runnerPackageDigest}
Runner build
{evalReport.run.runnerBuild}
runnerd
{evalReport.run.runnerdDigest}

Assertions

) : null} {tab === "timeline" ? (
{revisions.map((entry) => ( ))}
) : null} {tab === "state" ? (
) : null} {tab === "diff" ? (
{rows.map((row) =>
{row.path}{JSON.stringify(row.before)}{JSON.stringify(row.after)}
)} {rows.length === 0 ?

No changes between these revisions.

: null}
) : null} {tab === "documents" ? ( documents.length === 0 ?

No documents exist at r{selected.revision}.

: (
{documents.map((entry) => )}
{document !== null && visibleDocumentRevision !== null ? (
{document.key}

{document.title}

{visibleDocumentRevision.changeSummary ?

{visibleDocumentRevision.changeSummary}

: null}
{visibleDocumentRevision.body}
) : null}
) ) : null} {tab === "protocol" ?
{protocol.map((entry) =>
{entry.boundary} · {entry.event}
)}
: null} {tab === "runtime" ?
: null} {tab === "authority" ?
: null}
); }