feat(runner): invoke tools from devtools

This commit is contained in:
Dotta committed 2026-08-20 11:50:59 -05:00
1 parent 641dbd406c
commit 04b1aa2ac9
10 files changed
+226 -11

No files matched your search

@@ -362,10 +362,11 @@ test.describe("Capability issue thread", () => {
await expect(page.getByTestId("replay-next-turn")).toBeDisabled();
});
test("a terminal disposition disables the composer with a reason", async ({ page }) => {
test("a terminal disposition keeps the composer available", async ({ page }) => {
await open(page, "disposition-terminal");
await expect(page.locator('[data-thread-item="disposition"]')).toContainText("Done");
await expect(page.getByTestId("composer-reason")).toHaveText("Issue is done");
await expect(page.locator("#composer-input")).toBeEnabled();
await expect(page.getByText("This task is done. You can still continue the conversation.")).toBeVisible();
});
test("composer drafts survive a refresh", async ({ page }) => {
@@ -974,6 +974,13 @@ export function App() {
}
onClose={closeEvidence}
onJumpToThread={jumpToThread}
onInvokeTool={route.mode === "live" ? async (operationId, input) => {
setActionError(null);
const next = await capabilityLiveClient.invokeTool(snapshot.sessionId, operationId, input);
setSnapshot(next.view);
setSelectedTurnId(next.toolTurnId);
return next.toolResult;
} : undefined}
/>
) : null}
</div>
@@ -6,6 +6,7 @@ import type {
CapabilityEvidenceSectionId,
CapabilityEvidenceToolRow,
CapabilityIssueThreadSnapshot,
CapabilityJsonValue,
CapabilityToolDisposition,
} from "../../../src/issue-thread/types";
import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools";
@@ -122,6 +123,7 @@ export interface EvidencePanelProps {
onToggleSection: (section: CapabilityEvidenceSectionId) => void;
onClose: () => void;
onJumpToThread: (anchorId: string) => void;
onInvokeTool?: (operationId: string, input: CapabilityJsonValue) => Promise<CapabilityJsonValue>;
}
function sectionCount(
@@ -145,9 +147,29 @@ function sectionCount(
return String(rows.length);
}
function ToolsBrowser({ records }: { records: CapabilityEvidenceModel["tools"] }) {
function starterInput(schema: { properties?: Readonly<Record<string, { type?: string | readonly string[]; enum?: readonly CapabilityJsonValue[]; default?: CapabilityJsonValue }>>; required?: readonly string[] } | undefined): Record<string, CapabilityJsonValue> {
const result: Record<string, CapabilityJsonValue> = {};
for (const name of schema?.required ?? []) {
const property = schema?.properties?.[name];
if (property?.default !== undefined) result[name] = property.default;
else if (property?.enum?.[0] !== undefined) result[name] = property.enum[0];
else if (name === "idempotencyKey") result[name] = `devtools-${Date.now()}`;
else if (property?.type === "integer" || property?.type === "number") result[name] = 0;
else if (property?.type === "boolean") result[name] = false;
else if (property?.type === "array") result[name] = [];
else if (property?.type === "object") result[name] = {};
else result[name] = "";
}
return result;
}
function ToolsBrowser({ records, onInvoke }: { records: CapabilityEvidenceModel["tools"]; onInvoke?: (operationId: string, input: CapabilityJsonValue) => Promise<CapabilityJsonValue> }) {
const [collapsed, setCollapsed] = useState<CapabilityToolDisposition[]>([]);
const [selected, setSelected] = useState<CapabilityEvidenceToolRow | null>(null);
const [inputText, setInputText] = useState("{}");
const [invocationResult, setInvocationResult] = useState<CapabilityJsonValue | null>(null);
const [invocationError, setInvocationError] = useState<string | null>(null);
const [invoking, setInvoking] = useState(false);
const rows = useMemo(() => {
const unique = new Map<string, CapabilityEvidenceToolRow>();
for (const record of records) {
@@ -157,6 +179,13 @@ function ToolsBrowser({ records }: { records: CapabilityEvidenceModel["tools"] }
}, [records]);
const descriptor = selected === null ? undefined : capabilitySemanticToolDescriptor(selected.operationId);
useEffect(() => {
if (selected === null) return;
setInputText(JSON.stringify(starterInput(descriptor?.inputSchema), null, 2));
setInvocationResult(null);
setInvocationError(null);
}, [descriptor, selected]);
useEffect(() => {
if (selected === null) return;
const close = (event: KeyboardEvent) => { if (event.key === "Escape") setSelected(null); };
@@ -219,6 +248,32 @@ function ToolsBrowser({ records }: { records: CapabilityEvidenceModel["tools"] }
<h3 className="pit-tool-schema-title">Input schema</h3>
<pre className="pit-code">{JSON.stringify(descriptor?.inputSchema ?? {}, null, 2)}</pre>
</div>
{onInvoke !== undefined ? (
<div className="pit-tool-tester">
<h3 className="pit-tool-schema-title">Invoke in conversation</h3>
<p className="pit-muted">Runs through the real semantic dispatcher and records a turn, evidence, and company-state changes.</p>
<label className="pit-card-meta" htmlFor="tool-test-input">Parameters (JSON)</label>
<textarea id="tool-test-input" className="pit-tool-test-input" value={inputText} onChange={(event) => setInputText(event.target.value)} spellCheck={false} />
<button type="button" className="pit-button" disabled={invoking} onClick={() => {
let parsed: CapabilityJsonValue;
try {
parsed = JSON.parse(inputText) as CapabilityJsonValue;
} catch (error) {
setInvocationError(error instanceof Error ? error.message : String(error));
return;
}
setInvoking(true);
setInvocationError(null);
setInvocationResult(null);
void onInvoke(selected.operationId, parsed)
.then(setInvocationResult)
.catch((error) => setInvocationError(error instanceof Error ? error.message : String(error)))
.finally(() => setInvoking(false));
}}>{invoking ? "Invoking…" : "Invoke as tool call"}</button>
{invocationError !== null ? <p className="pit-error" role="alert">{invocationError}</p> : null}
{invocationResult !== null ? <pre className="pit-code" data-testid="tool-test-result">{JSON.stringify(invocationResult, null, 2)}</pre> : null}
</div>
) : null}
</div>
</div>
) : null}
@@ -285,6 +340,7 @@ export function EvidencePanel(props: EvidencePanelProps) {
onToggleSection,
onClose,
onJumpToThread,
onInvokeTool,
} = props;
const headingRef = useRef<HTMLHeadingElement | null>(null);
const evidence = snapshot.evidence;
@@ -353,7 +409,7 @@ export function EvidencePanel(props: EvidencePanelProps) {
open={isOpen("tools")}
onToggle={() => onToggleSection("tools")}
>
<ToolsBrowser records={filterByTurn(evidence.tools, selectedTurnId)} />
<ToolsBrowser records={filterByTurn(evidence.tools, selectedTurnId)} onInvoke={onInvokeTool} />
</Section>
<Section
@@ -1516,6 +1516,28 @@ textarea {
overflow-wrap: anywhere;
}
.pit-tool-tester {
display: flex;
flex-direction: column;
gap: var(--pit-space-2);
padding-top: var(--pit-space-4);
border-top: 1px solid var(--pit-border-strong);
}
.pit-tool-test-input {
box-sizing: border-box;
width: 100%;
min-height: 10rem;
resize: vertical;
padding: var(--pit-space-3);
border: 1px solid var(--pit-border-strong);
border-radius: var(--pit-radius-sm);
background: var(--pit-background);
color: var(--pit-foreground);
font-family: var(--pit-font-mono);
font-size: var(--pit-text-xs);
}
.pit-verdict {
display: inline-flex;
align-items: center;
@@ -1,5 +1,6 @@
import type { CapabilityIssueThreadSnapshot } from "../../../src/issue-thread/types";
import type { CapabilityDevtoolsSnapshot } from "../../../src/devtools";
import type { CapabilityJsonValue } from "../../../src/mock-core/capability-control-plane-types";
import {
CAPABILITY_TURN_STREAM_ACCEPT,
CapabilityTurnStreamError,
@@ -50,6 +51,11 @@ export interface CapabilityLiveResponse {
turns?: number;
}
export interface CapabilityToolTestResponse extends CapabilityLiveResponse {
toolResult: CapabilityJsonValue;
toolTurnId: string;
}
/**
* The server names its own failures. Surfacing the code lets the clean room say
* "this chat hit its turn limit" instead of "500", and — more importantly —
@@ -137,6 +143,13 @@ export const capabilityLiveClient = {
fork(sessionId: string, revision: number): Promise<CapabilityLiveResponse> {
return post("/devtools/fork", { sessionId, revision });
},
invokeTool(
sessionId: string,
operationId: string,
input: CapabilityJsonValue,
): Promise<CapabilityToolTestResponse> {
return post("/tool", { sessionId, operationId, input }) as Promise<CapabilityToolTestResponse>;
},
async load(sessionId: string | null): Promise<CapabilityLiveResponse> {
const query = sessionId === null ? "" : `?sessionId=${encodeURIComponent(sessionId)}`;
const response = await fetch(`${BASE}/session${query}`, { credentials: CREDENTIALS });
@@ -722,7 +722,19 @@ export function createCapabilityIssueThreadMiddleware(options = {}) {
return;
}
if (route === "message") {
if (route === "tool") {
const invocation = await entry.session.invokeTool(
String(body.operationId ?? ""),
body.input ?? {},
);
const projected = payload(runner, entry);
send(response, 200, {
...projected,
toolResult: invocation.result,
toolTurnId: projected.view.turns.at(-1)?.id ?? null,
});
return;
} else if (route === "message") {
const message = String(body.message ?? "");
if (Buffer.byteLength(message, "utf8") > MAX_MESSAGE_BYTES) {
throw new RouteError(413, "message_too_large", "Message exceeds the server limit.");
@@ -996,9 +996,9 @@ const BUILDERS: Record<CapabilityUiShotSlug, () => CapabilityIssueThreadSnapshot
threadAnchorId: "item-disposition-1",
});
snapshot.composer = {
state: "disabled",
helper: null,
reason: "Issue is done",
state: "ready",
helper: "This task is done. You can still continue the conversation.",
reason: null,
pendingInteractionId: null,
};
return snapshot;
@@ -526,11 +526,19 @@ function composerModel(
pendingInteractionId,
};
}
if (terminal || snapshot.status === "closed" || snapshot.status === "failed") {
if (snapshot.status === "closed" || snapshot.status === "failed") {
return {
state: "disabled",
helper: null,
reason: terminal ? "Issue is done" : "Session is closed",
reason: "Session is closed",
pendingInteractionId: null,
};
}
if (terminal) {
return {
state: "ready",
helper: "This task is done. You can still continue the conversation.",
reason: null,
pendingInteractionId: null,
};
}
@@ -672,6 +672,46 @@ describe("Capability clean-room chat server", () => {
expect(after.view.evidence.state.length).toBeGreaterThan(0);
});
it("invokes a DevTools tool as a durable conversation turn", async () => {
const opened = await call("/api/capability/ui/cleanroom/session");
const after = await call("/api/capability/ui/tool", {
sessionId: opened.sessionId,
operationId: "report_progress",
input: {
idempotencyKey: "devtools-report-1",
body: "Progress entered manually from DevTools.",
},
});
expect(after.toolResult).toMatchObject({ ok: true, operationId: "report_progress" });
expect(after.toolTurnId).toMatch(/^turn-/);
const turn = after.view.turns.find((candidate: { id: string }) => candidate.id === after.toolTurnId);
expect(turn).toBeDefined();
expect(turn.items.some((item: CapabilityThreadItem) => item.kind === "user_message")).toBe(true);
expect(turn.items.some((item: CapabilityThreadItem) => item.kind === "tool_activity" && item.operationId === "report_progress")).toBe(true);
expect(turn.items.some((item: CapabilityThreadItem) => item.kind === "durable_comment" && item.body === "Progress entered manually from DevTools.")).toBe(true);
expect(after.view.evidence.state.at(-1)?.toRevision).toBeGreaterThan(0);
});
it("continues the provider conversation after the task is finished", async () => {
const opened = await call("/api/capability/ui/cleanroom/session");
const finished = await call("/api/capability/ui/tool", {
sessionId: opened.sessionId,
operationId: "finish_task",
input: { idempotencyKey: "devtools-finish-1", summary: "The mock task is complete." },
});
expect(finished.view.issue.status).toBe("done");
expect(finished.view.composer.state).toBe("ready");
const continued = await call("/api/capability/ui/message", {
sessionId: opened.sessionId,
message: "Explain what was completed.",
});
expect(items(continued.view).some(
(item) => item.kind === "user_message" && item.body === "Explain what was completed.",
)).toBe(true);
});
it("exposes full state history and can fork a retained revision", async () => {
const opened = await call("/api/capability/ui/cleanroom/session");
await call("/api/capability/ui/message", {
@@ -1070,7 +1110,7 @@ describe("Capability clean-room chat server", () => {
}
// Every mutation route, not only the ones that create sessions.
const mutations = ["message", "interrupt", "reconnect", "reset", "interaction"];
const mutations = ["message", "tool", "interrupt", "reconnect", "reset", "interaction"];
for (const route of mutations) {
expect(
await status(`/api/capability/ui/${route}`, { sessionId: aliceRoom.sessionId, message: "hi" }, mallory),
@@ -209,6 +209,12 @@ export interface CapabilityLiveTurnResult {
snapshot: CapabilityLiveSessionSnapshot;
}
export interface CapabilityLiveToolInvocationResult {
turnId: string;
result: CapabilitySemanticToolResult;
snapshot: CapabilityLiveSessionSnapshot;
}
/**
* Live turn events (track 7Q).
*
@@ -1104,6 +1110,56 @@ export class CapabilityLiveSession {
return { ...result, snapshot: this.snapshot() };
}
/**
* Runs an operator-authored DevTools call through the exact dispatcher used
* by Codex. It receives a real turn id, durable call/result evidence, and a
* state-history entry so it is replayable as part of the conversation rather
* than an out-of-band API console.
*/
async invokeTool(operationId: string, input: unknown): Promise<CapabilityLiveToolInvocationResult> {
if (this.#status === "closed" || this.#status === "failed") {
throw new Error("Capability live session is not connected");
}
if (this.#activeTurnId !== null || this.#turnWaiter !== null) {
throw new Error("Capability live session already has an active turn");
}
const turnId = `devtools-${randomUUID()}`;
const callId = `devtools-call-${randomUUID()}`;
const argumentsValue = jsonValue(input);
const beforeRevision = this.#port.snapshot().revision;
this.#appendTranscript("user", `DevTools invoked \`${operationId}\`.`, turnId);
this.#appendEvidence("tool_call", turnId, {
callId,
operationId,
input: argumentsValue,
beforeRevision,
});
const result = await this.#dispatcher.dispatch({
runId: this.#authority.runId,
callId,
operationId,
input: argumentsValue,
});
this.#recordStateRevision(turnId, operationId);
this.#appendEvidence("tool_result", turnId, {
callId,
operationId,
result: jsonValue(result),
beforeRevision,
afterRevision: result.stateRevision,
});
this.#appendTranscript(
"assistant",
result.ok
? `DevTools call \`${operationId}\` completed at company state revision ${result.stateRevision}.`
: `DevTools call \`${operationId}\` was denied: ${result.denial.message}`,
turnId,
);
this.#recordTerminalFact(turnId, "completed");
await this.#persist();
return { turnId, result, snapshot: this.snapshot() };
}
#armTurnWaiter(): Promise<Omit<CapabilityLiveTurnResult, "snapshot">> {
if (this.#turnWaiter !== null) throw new Error("Capability live session already has a turn waiter");
return new Promise<Omit<CapabilityLiveTurnResult, "snapshot">>((resolve, reject) => {