import { useEffect, useMemo, useState } from "react";
import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools";
import { Icon, type IconName } from "./Icons";
import { EvalAssertions, type EvalAssertion } from "./ThreadItems";
export type CapabilityDevtoolsTab =
| "eval"
| "evidence"
| "timeline"
| "state"
| "diff"
| "documents"
| "protocol"
| "trace"
| "runtime"
| "authority";
type Json = null | boolean | number | string | Json[] | { [key: string]: Json };
export interface EvalInspectorReport {
attemptId: string;
caseId: string;
disposition: string;
passed: boolean;
checks: EvalAssertion[];
publication?: { schema: string; notice: string };
run: {
model: string;
provider: string;
driver: string;
providerVersion: string | null;
runnerProvider: string;
configuration: string;
sessionId: string;
providerSessionId: string | null;
agentVersion: string | null;
retainedSession: boolean | null;
retainedSessionStatus: string | null;
fixtureDigest: string;
runnerPackageDigest: string;
runnerdDigest: string;
runnerBuild: string;
startedAt: string;
finishedAt: string;
durationMs: number | null;
initialRevision: number | null;
finalRevision: number | null;
usage: {
agentTurns: number;
providerRequests: number | null;
inputTokens?: number;
outputTokens?: number;
cachedInputTokens?: number;
reasoningTokens?: number;
providerReportedCostNanodollars?: number;
estimatedCostNanodollars?: number;
pricingVersion?: string;
} | null;
};
}
function JsonTree({ value, name = "root" }: { value: Json; name?: string }) {
if (value === null || typeof value !== "object") {
return {JSON.stringify(value)};
}
const entries = Array.isArray(value)
? value.map((entry, index) => [String(index), entry] as const)
: Object.entries(value);
return (
{name}{" "}
{Array.isArray(value) ? `[${entries.length}]` : `{${entries.length}}`}
{entries.map(([key, entry]) => (
{key}
{entry !== null && typeof entry === "object" ? (
) : (
)}
))}
);
}
interface DiffRow {
path: string;
before: Json | undefined;
after: Json | undefined;
}
function diff(
before: Json | undefined,
after: Json | undefined,
path = "state",
): DiffRow[] {
if (JSON.stringify(before) === JSON.stringify(after)) return [];
if (
before !== null &&
after !== null &&
typeof before === "object" &&
typeof after === "object" &&
!Array.isArray(before) &&
!Array.isArray(after)
) {
const keys = new Set([...Object.keys(before), ...Object.keys(after)]);
return [...keys].flatMap((key) =>
diff(before[key], after[key], `${path}.${key}`),
);
}
return [{ path, before, after }];
}
function download(snapshot: CapabilityDevtoolsSnapshot) {
const blob = new Blob([JSON.stringify(snapshot, null, 2)], {
type: "application/json",
});
const url = URL.createObjectURL(blob);
const anchor = document.createElement("a");
anchor.href = url;
anchor.download = `paperclip-devtools-r${snapshot.currentRevision}.json`;
anchor.click();
URL.revokeObjectURL(url);
}
const TABS: ReadonlyArray<{
id: CapabilityDevtoolsTab;
icon: IconName;
label: string;
}> = [
{ id: "evidence", icon: "evidence", label: "Evidence" },
{ id: "timeline", icon: "timeline", label: "Timeline" },
{ id: "state", icon: "state", label: "State" },
{ id: "diff", icon: "diff", label: "Diff" },
{ id: "documents", icon: "documents", label: "Documents" },
{ id: "protocol", icon: "protocol", label: "Protocol" },
{ id: "trace", icon: "protocol", label: "Provider trace" },
{ id: "runtime", icon: "terminal", label: "Runtime" },
{ id: "authority", icon: "authority", label: "Authority" },
];
function documentsOf(state: Json): Array<{
id: string;
key: string;
title: string;
revisions: Array<{
id: string;
revision: number;
body: string;
changeSummary: string | null;
createdAt: string;
}>;
}> {
if (state === null || typeof state !== "object" || Array.isArray(state))
return [];
const documents = state.documents;
if (!Array.isArray(documents)) return [];
return documents.flatMap((value) => {
if (value === null || typeof value !== "object" || Array.isArray(value))
return [];
const revisions = Array.isArray(value.revisions)
? value.revisions.flatMap((candidate) => {
if (
candidate === null ||
typeof candidate !== "object" ||
Array.isArray(candidate)
)
return [];
return [
{
id: String(candidate.id ?? ""),
revision: Number(candidate.revision ?? 0),
body: String(candidate.body ?? ""),
changeSummary:
candidate.changeSummary === null
? null
: String(candidate.changeSummary ?? ""),
createdAt: String(candidate.createdAt ?? ""),
},
];
})
: [];
return [
{
id: String(value.id ?? ""),
key: String(value.key ?? ""),
title: String(value.title ?? "Untitled"),
revisions,
},
];
});
}
export function EvalReportInspector({
evalReport,
}: {
evalReport: EvalInspectorReport;
}) {
const isPublic = Boolean(evalReport.publication);
return (
← Eval suite
{evalReport.passed
? "PASS"
: evalReport.disposition.replaceAll("_", " ").toUpperCase()}
{evalReport.attemptId}
- Model
-
{evalReport.run.model.startsWith(`${evalReport.run.provider}/`)
? evalReport.run.model
: `${evalReport.run.provider}/${evalReport.run.model}`}
- Configuration
- {evalReport.run.configuration}
- Session
-
{isPublic
? "Withheld from public replay"
: evalReport.run.sessionId}
- Provider session
-
{isPublic
? "Withheld from public replay"
: (evalReport.run.providerSessionId ?? "unavailable")}
- Driver
-
{evalReport.run.driver}
{evalReport.run.providerVersion
? ` · ${evalReport.run.providerVersion}`
: ""}
{evalReport.run.agentVersion ? (
- Agent version
- {evalReport.run.agentVersion}
) : null}
- Retained session
-
{isPublic
? "Withheld from public replay"
: evalReport.run.retainedSession === true
? (evalReport.run.retainedSessionStatus ?? "retained")
: "not applicable"}
- Duration
-
{evalReport.run.durationMs == null
? "unavailable"
: `${evalReport.run.durationMs} ms`}
- Fixture
- {evalReport.run.fixtureDigest}
- State
-
{isPublic
? "Withheld from public replay"
: evalReport.run.initialRevision == null || evalReport.run.finalRevision == null
? "unavailable"
: `r${evalReport.run.initialRevision} → r${evalReport.run.finalRevision}`}
- Tokens
-
{evalReport.run.usage == null
? "unavailable"
: `${evalReport.run.usage.inputTokens ?? "unavailable"} in · ${evalReport.run.usage.outputTokens ?? "unavailable"} out · ${evalReport.run.usage.cachedInputTokens ?? "unavailable"} cached`}
- Agent turns
- {evalReport.run.usage?.agentTurns ?? "unknown"}
- Provider requests
- {evalReport.run.usage?.providerRequests ?? "unavailable"}
- Estimated cost
-
{evalReport.run.usage?.estimatedCostNanodollars == null
? "unavailable"
: `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)}${evalReport.run.usage.pricingVersion ? ` · ${evalReport.run.usage.pricingVersion}` : ""}`}
- Provider list cost
-
{typeof evalReport.run.usage?.providerReportedCostNanodollars !==
"number"
? "unknown"
: `$${(evalReport.run.usage.providerReportedCostNanodollars / 1_000_000_000).toFixed(6)}`}
- Runner
- {evalReport.run.runnerPackageDigest}
- Runner build
- {evalReport.run.runnerBuild}
- runnerd
- {evalReport.run.runnerdDigest}
Assertions
);
}
export function DevtoolsInspector({
snapshot,
onFork,
tab,
onTabChange,
evalReport,
}: {
snapshot: CapabilityDevtoolsSnapshot;
onFork: (revision: number) => void;
tab: CapabilityDevtoolsTab;
onTabChange: (tab: CapabilityDevtoolsTab) => void;
evalReport?: EvalInspectorReport | null;
}) {
const [query, setQuery] = useState("");
const latest = snapshot.revisions.at(-1)!;
const [revision, setRevision] = useState(latest.revision);
const [following, setFollowing] = useState(true);
const [compareRevision, setCompareRevision] = useState(
snapshot.revisions[0]!.revision,
);
const selected =
snapshot.revisions.find((entry) => entry.revision === revision) ?? latest;
const compared =
snapshot.revisions.find((entry) => entry.revision === compareRevision) ??
snapshot.revisions[0]!;
const rows = useMemo(
() => diff(compared.state as Json, selected.state as Json),
[compared.state, selected.state],
);
const normalized = query.trim().toLowerCase();
const revisions = snapshot.revisions.filter(
(entry) =>
normalized.length === 0 ||
JSON.stringify(entry).toLowerCase().includes(normalized),
);
const protocol = snapshot.protocol.filter(
(entry) =>
normalized.length === 0 ||
JSON.stringify(entry).toLowerCase().includes(normalized),
);
const documents = useMemo(
() => documentsOf(selected.state as Json),
[selected.state],
);
const [documentId, setDocumentId] = useState("");
const document =
documents.find((entry) => entry.id === documentId) ?? documents[0] ?? null;
const [documentRevision, setDocumentRevision] = useState(0);
const visibleDocumentRevision =
document?.revisions.find((entry) => entry.revision === documentRevision) ??
document?.revisions.at(-1) ??
null;
useEffect(() => {
if (following) setRevision(latest.revision);
}, [following, latest.revision]);
return (
State monitor
setQuery(event.target.value)}
placeholder="operation, entity, turn…"
/>
{[
...(evalReport
? [
{
id: "eval" as const,
icon: "evidence" as const,
label: "Eval",
},
]
: []),
...TABS,
].map(({ id, icon, label }) => (
))}
{tab === "eval" && evalReport ? (
) : null}
{tab === "timeline" ? (
{revisions.map((entry) => (
))}
) : null}
{tab === "state" ? (
) : null}
{tab === "diff" ? (
{rows.map((row) => (
{row.path}
{JSON.stringify(row.before)}
{JSON.stringify(row.after)}
))}
{rows.length === 0 ? (
No changes between these revisions.
) : null}
) : null}
{tab === "documents" ? (
documents.length === 0 ? (
▤
No documents exist at r{selected.revision}.
) : (
{documents.map((entry) => (
))}
{document !== null && visibleDocumentRevision !== null ? (
{visibleDocumentRevision.changeSummary ? (
{visibleDocumentRevision.changeSummary}
) : null}
{visibleDocumentRevision.body}
) : null}
)
) : null}
{tab === "protocol" ? (
{protocol.map((entry) => (
{entry.boundary} · {entry.event}
))}
) : null}
{tab === "trace" ? (
snapshot.providerTrace ? (
{snapshot.providerTrace.status} · expires{" "}
{snapshot.providerTrace.expiresAt}
Raw frame → Interpretation → PRP events → Presentation
) : (
◇
Raw provider capture was off for this run. Canonical protocol
records remain available in Protocol.
)
) : null}
{tab === "runtime" ? (
) : null}
{tab === "authority" ? (
) : null}
);
}