feat(runner): add semantic tool detail browser

This commit is contained in:
Dotta committed 2026-08-20 10:55:43 -05:00
1 parent f5f5997cb8
commit 61b17641a7
4 files changed
+228 -38

No files matched your search

@@ -240,6 +240,22 @@ test.describe("Capability issue thread", () => {
await expect(panel.getByText("Agent tool — granted")).toBeVisible();
});
test("tool exposure groups are deduplicated, foldable, and open catalog details", async ({ page }) => {
await open(page, "debug-panel-open", { panel: "tools" });
const panel = page.getByTestId("evidence-panel");
const always = panel.getByRole("button", { name: /Agent tool — always/ });
await expect(always).toHaveCount(1);
await always.click();
await expect(panel.getByRole("button", { name: /get_task_context/ })).toHaveCount(0);
await always.click();
await panel.getByRole("button", { name: /get_task_context/ }).click();
const dialog = page.getByRole("dialog", { name: "Get active task context" });
await expect(dialog).toContainText("Read the active mock task, actor, wake, ancestors, budget, and interaction results.");
await expect(dialog).toContainText("Input schema");
await dialog.getByRole("button", { name: "Close tool details" }).click();
await expect(dialog).toHaveCount(0);
});
test("Evidence records link back to their thread anchor", async ({ page }) => {
await open(page, "debug-panel-open", { panel: "calls" });
const panel = page.getByTestId("evidence-panel");
@@ -1,15 +1,17 @@
import { useEffect, useRef, type ReactNode } from "react";
import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
import type {
CapabilityEvidenceModel,
CapabilityEvidenceRunnerRecord,
CapabilityEvidenceSectionId,
CapabilityEvidenceToolRow,
CapabilityIssueThreadSnapshot,
CapabilityToolDisposition,
} from "../../../src/issue-thread/types";
import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools";
import { DevtoolsInspector } from "./DevtoolsInspector";
import { Icon } from "./Icons";
import { capabilitySemanticToolDescriptor } from "../../../src/semantic-tools/catalog";
import {
CAPABILITY_EVIDENCE_SECTIONS,
capabilityDispositionLabel,
@@ -127,6 +129,10 @@ function sectionCount(
section: CapabilityEvidenceSectionId,
turnId: string | "all",
): string {
if (section === "tools") {
const records = filterByTurn(evidence.tools, turnId);
return String(new Set(records.flatMap((record) => record.rows.map((row) => row.operationId))).size);
}
if (section === "parity") {
const rows = filterByTurn(evidence.parity, turnId);
const passing = rows.filter((row) => row.verdict === "pass").length;
@@ -139,6 +145,87 @@ function sectionCount(
return String(rows.length);
}
function ToolsBrowser({ records }: { records: CapabilityEvidenceModel["tools"] }) {
const [collapsed, setCollapsed] = useState<CapabilityToolDisposition[]>([]);
const [selected, setSelected] = useState<CapabilityEvidenceToolRow | null>(null);
const rows = useMemo(() => {
const unique = new Map<string, CapabilityEvidenceToolRow>();
for (const record of records) {
for (const row of record.rows) unique.set(row.operationId, row);
}
return [...unique.values()];
}, [records]);
const descriptor = selected === null ? undefined : capabilitySemanticToolDescriptor(selected.operationId);
useEffect(() => {
if (selected === null) return;
const close = (event: KeyboardEvent) => { if (event.key === "Escape") setSelected(null); };
document.addEventListener("keydown", close);
return () => document.removeEventListener("keydown", close);
}, [selected]);
return (
<>
<div className="pit-tool-groups">
{DISPOSITION_ORDER.map((disposition) => {
const group = rows.filter((row) => row.disposition === disposition);
if (group.length === 0) return null;
const folded = collapsed.includes(disposition);
const title = disposition === "control_plane_owned"
? "Control plane (not exposed to the agent)"
: capabilityDispositionLabel(disposition);
return (
<section className="pit-tool-group" data-group={disposition} key={disposition}>
<h4>
<button type="button" className="pit-tool-group-toggle" aria-expanded={!folded} onClick={() => setCollapsed((current) => current.includes(disposition) ? current.filter((entry) => entry !== disposition) : [...current, disposition])}>
<span aria-hidden="true">{folded ? "›" : "⌄"}</span>
<span>{title}</span>
<span className="pit-section-count">{group.length}</span>
</button>
</h4>
{!folded ? (
<div className="pit-tool-group-rows">
{group.map((row) => (
<button type="button" className="pit-tool-row" key={row.operationId} data-disposition={disposition} onClick={() => setSelected(row)}>
<span className="pit-record-title">{row.operationId}</span>
<span className="pit-tool-row-description">{row.description}</span>
{row.grant !== null && disposition === "optional_agent_tool" ? <span className="pit-grant">{row.grant}</span> : null}
<span className="pit-activity-caret" aria-hidden="true">›</span>
</button>
))}
</div>
) : null}
</section>
);
})}
</div>
{selected !== null ? (
<div className="pit-dialog-backdrop" onMouseDown={(event) => { if (event.target === event.currentTarget) setSelected(null); }}>
<div className="pit-dialog pit-tool-dialog" role="dialog" aria-modal="true" aria-labelledby="tool-dialog-title">
<header className="pit-tool-dialog-head">
<div><span className="pit-card-meta">Semantic tool · v{descriptor?.version ?? 1}</span><h2 id="tool-dialog-title">{descriptor?.title ?? selected.operationId}</h2></div>
<button type="button" className="pit-icon-button" aria-label="Close tool details" title="Close tool details" onClick={() => setSelected(null)}><Icon name="close" /></button>
</header>
<code className="pit-tool-operation">{selected.operationId}</code>
<p>{descriptor?.description ?? selected.description}</p>
<dl className="pit-tool-facts">
<div><dt>Placement</dt><dd>{capabilityDispositionLabel(selected.disposition)}</dd></div>
<div><dt>Required claims</dt><dd>{descriptor?.requiredClaims.join(", ") || selected.grant || "None"}</dd></div>
<div><dt>Task modes</dt><dd>{descriptor?.allowedModes.join(", ") || "—"}</dd></div>
<div><dt>Actor roles</dt><dd>{descriptor?.allowedRoles?.join(", ") || "Any allowed role"}</dd></div>
</dl>
<div>
<h3 className="pit-tool-schema-title">Input schema</h3>
<pre className="pit-code">{JSON.stringify(descriptor?.inputSchema ?? {}, null, 2)}</pre>
</div>
</div>
</div>
) : null}
</>
);
}
function filterByTurn<T extends { turnId: string }>(
rows: ReadonlyArray<T>,
turnId: string | "all",
@@ -264,35 +351,7 @@ export function EvidencePanel(props: EvidencePanelProps) {
open={isOpen("tools")}
onToggle={() => onToggleSection("tools")}
>
{filterByTurn(evidence.tools, selectedTurnId).map((record) => (
<div key={record.id}>
{DISPOSITION_ORDER.map((disposition) => {
const rows = record.rows.filter((row) => row.disposition === disposition);
if (rows.length === 0) return null;
return (
<div key={disposition}>
<p className="pit-tool-group-label" data-group={disposition}>
{disposition === "control_plane_owned"
? "Control plane (not exposed to the agent)"
: capabilityDispositionLabel(disposition)}
</p>
{rows.map((row) => (
<p
className="pit-tool-row"
key={row.operationId}
data-disposition={disposition}
>
<span className="pit-record-title">{row.operationId}</span>
{row.grant !== null && disposition === "optional_agent_tool" ? (
<span className="pit-grant">{row.grant}</span>
) : null}
</p>
))}
</div>
);
})}
</div>
))}
<ToolsBrowser records={filterByTurn(evidence.tools, selectedTurnId)} />
</Section>
<Section
@@ -1342,23 +1342,63 @@ textarea {
background: var(--pit-surface-sunken);
}
.pit-tool-group-label {
font-size: var(--pit-text-xs);
font-weight: 600;
color: var(--pit-muted-foreground);
margin-top: var(--pit-space-2);
.pit-tool-groups {
display: grid;
gap: var(--pit-space-5);
padding: var(--pit-space-3);
}
.pit-tool-group-label[data-group="control_plane_owned"] {
.pit-tool-group h4 {
margin: 0;
}
.pit-tool-group-toggle {
display: flex;
align-items: center;
gap: var(--pit-space-2);
width: 100%;
min-height: var(--pit-touch-target);
padding: 0;
border: 0;
border-bottom: 1px solid var(--pit-border);
background: transparent;
color: var(--pit-foreground);
font-size: var(--pit-text-sm);
font-weight: 700;
text-align: left;
cursor: pointer;
}
.pit-tool-group[data-group="control_plane_owned"] .pit-tool-group-toggle {
color: var(--pit-faint-foreground);
}
.pit-tool-group-rows {
display: grid;
padding-top: var(--pit-space-2);
}
.pit-tool-row {
display: flex;
display: grid;
grid-template-columns: minmax(9rem, 0.55fr) minmax(0, 1fr) auto auto;
gap: var(--pit-space-2);
align-items: baseline;
align-items: center;
width: 100%;
min-height: var(--pit-touch-target);
padding: var(--pit-space-2);
border: 0;
border-bottom: 1px solid var(--pit-border);
background: transparent;
color: inherit;
font-size: var(--pit-text-xs);
text-align: left;
min-width: 0;
cursor: pointer;
}
.pit-tool-row:hover,
.pit-tool-row:focus-visible {
background: var(--pit-accent-surface);
}
.pit-tool-row[data-disposition="control_plane_owned"] {
@@ -1371,6 +1411,63 @@ textarea {
overflow-wrap: anywhere;
}
.pit-tool-row-description {
color: var(--pit-muted-foreground);
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.pit-tool-dialog {
width: min(44rem, 100%);
max-height: calc(100vh - var(--pit-space-6));
overflow: auto;
}
.pit-tool-dialog-head {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: var(--pit-space-3);
}
.pit-tool-dialog-head h2,
.pit-tool-schema-title {
margin: 0;
}
.pit-tool-operation {
align-self: flex-start;
padding: var(--pit-space-1) var(--pit-space-2);
border: 1px solid var(--pit-live);
border-radius: var(--pit-radius-sm);
background: var(--pit-live-surface);
color: var(--pit-foreground);
}
.pit-tool-facts {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: var(--pit-space-3);
margin: 0;
}
.pit-tool-facts div {
padding: var(--pit-space-2);
border: 1px solid var(--pit-border);
border-radius: var(--pit-radius-sm);
}
.pit-tool-facts dt {
color: var(--pit-muted-foreground);
font-size: var(--pit-text-xs);
}
.pit-tool-facts dd {
margin: var(--pit-space-1) 0 0;
overflow-wrap: anywhere;
}
.pit-verdict {
display: inline-flex;
align-items: center;
@@ -1898,6 +1995,19 @@ textarea {
border-right: 0;
border-bottom: 1px solid var(--pit-border);
}
.pit-tool-row {
grid-template-columns: minmax(0, 1fr) auto;
}
.pit-tool-row-description {
grid-column: 1 / -1;
white-space: normal;
}
.pit-tool-facts {
grid-template-columns: minmax(0, 1fr);
}
}
.pit-diff-row del {
@@ -119,6 +119,11 @@ boundary records, runner/runtime facts, and effective authority. A dedicated
**Documents** tab renders every mock document and lets the operator switch
between its retained revisions without digging through raw JSON.
The Evidence log consolidates repeated tool-exposure snapshots into one
deduplicated catalog. Its Always, Granted, and Control-plane groups fold
independently; selecting a tool opens its catalog title, description, placement,
required claims, allowed task modes and roles, and full input schema.
**Pause** pins a revision while work continues, **Export** downloads the
redacted DevTools snapshot, and **Fork rN** retires the current mock session and
starts a new executable branch from that retained state. Provider payloads,