fix(runner): scope streaming chat evidence

Hold staged replay on a partial model message, limit activity to revealed records, and visibly explain disabled controls. Re-record the two streaming acceptance shots.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-08-09 16:21:25 +00:00
1 parent 362754a633
commit c8422d1aaa
10 files changed
+250 -34

No files matched your search

@@ -161,9 +161,36 @@ test.describe("determinism and evidence", () => {
await open(page, "#/chat/ap-mcp-gate-01?replay=fake&stage=streaming", "streaming");
await expect(page.locator('[data-slot="composer"]')).toHaveAttribute("data-state", "active-turn");
await expect(page.locator('[data-testid^="turn-pending-"]')).toBeVisible();
const modelCard = page.locator('[data-testid="streaming-model-card"]');
await expect(modelCard).toBeVisible();
await expect(modelCard.locator(".pcr-stream-cursor")).toBeVisible();
const activity = page.locator('[data-testid="activity-turn-2"]');
await expect(activity.locator('[data-testid="activity-turn-header-2"]')).toContainText(
"Turn 2 · running",
);
const calls = activity.locator('[data-testid="activity-section-2-calls"]');
await expect(calls).toContainText("request_approval");
await expect(calls).not.toContainText("request_review");
await expect(activity.locator('[data-testid="activity-section-2-authorization"]')).toContainText(
"Authorization (1)",
);
await expect(activity.locator('[data-testid="activity-section-2-diff"]')).toHaveCount(0);
await expect(activity.locator('[data-testid="activity-section-2-parity"]')).toHaveCount(0);
await expect(page.locator('[data-testid="activity-session"]')).toHaveCount(0);
// Controls that would corrupt the running turn state their reason.
await expect(page.locator('[data-testid="reset-session"]')).toBeDisabled();
await expect(page.locator('[data-testid="replay-session"]')).toBeDisabled();
const reason = page.locator('[data-testid="running-controls-reason"]');
await expect(reason).toHaveText("Controls return when the turn settles.");
for (const testId of [
"chat-mode-scripted",
"chat-mode-codex",
"reset-session",
"replay-session",
]) {
const control = page.locator(`[data-testid="${testId}"]`);
await expect(control).toBeDisabled();
await expect(control).toHaveAttribute("aria-describedby", "running-controls-reason");
}
});
test("no credential or raw secret reaches the chat page", async ({ page }) => {
@@ -108,6 +108,7 @@ export function ExplorerApp({
const [highlight, setHighlight] = React.useState<number | null>(null);
const [runFocusRequest, setRunFocusRequest] = React.useState(0);
const [chatArtifact, setChatArtifact] = React.useState<Phase7ChatSessionArtifact | null>(null);
const [chatRevealedSequence, setChatRevealedSequence] = React.useState(0);
const [chatState, setChatState] = React.useState<Phase7ChatState>("seeding");
const [stripFocusRequest, setStripFocusRequest] = React.useState({ turn: 0, nonce: 0 });
const shellRef = React.useRef<HTMLDivElement>(null);
@@ -387,6 +388,7 @@ export function ExplorerApp({
onOpenTurn={openTurn}
onOpenParity={() => openTurn(chatArtifact?.turns.at(-1)?.turn ?? 0)}
onArtifactChange={setChatArtifact}
onRevealSequenceChange={setChatRevealedSequence}
onStateChange={setChatState}
onCancelScenarioSwitch={(entryId) => navigate({ ...route, caseId: entryId })}
openSession={openSession}
@@ -455,6 +457,7 @@ export function ExplorerApp({
) : (
<ActivityStream
artifact={chatArtifact}
revealedSequence={chatRevealedSequence}
filter={route.activityFilter}
onFilterChange={(filter: Phase7ActivityFilter) =>
navigate({ ...route, activityFilter: filter })
@@ -53,22 +53,32 @@ function sessionFor(id: string): Phase7ChatSessionArtifact {
return artifact;
}
function conversation(id: string, revealed = Number.MAX_SAFE_INTEGER): string {
function conversation(
id: string,
revealed = Number.MAX_SAFE_INTEGER,
runningTurn: number | null = null,
): string {
return renderToStaticMarkup(
<ChatConversation
artifact={sessionFor(id)}
revealedSequence={revealed}
runningTurn={null}
runningTurn={runningTurn}
activeTurn={null}
onOpenTurn={() => undefined}
/>,
);
}
function activity(id: string, filter: Parameters<typeof ActivityStream>[0]["filter"] = "all", openTurn: number | null = null): string {
function activity(
id: string,
filter: Parameters<typeof ActivityStream>[0]["filter"] = "all",
openTurn: number | null = null,
revealedSequence = Number.MAX_SAFE_INTEGER,
): string {
return renderToStaticMarkup(
<ActivityStream
artifact={sessionFor(id)}
revealedSequence={revealedSequence}
filter={filter}
onFilterChange={() => undefined}
openTurn={openTurn}
@@ -123,6 +133,21 @@ describe("chat conversation", () => {
// Earlier turns are untouched by the reveal of a later one.
expect(markup).toContain('data-testid="turn-activity-strip-1"');
});
it("holds the staged turn on a visibly streaming model card", () => {
const artifact = sessionFor("ap-mcp-gate-01");
const lastTurn = artifact.turns.at(-1)!;
const model = artifact.timeline.find(
(entry) => entry.turn === lastTurn.turn && entry.kind === "agent_message",
);
if (model?.kind !== "agent_message") throw new Error("staged turn has no model message");
const markup = conversation("ap-mcp-gate-01", model.sequence, lastTurn.turn);
expect(markup).toContain('data-testid="streaming-model-card"');
expect(markup).toContain('data-state="streaming"');
expect(markup).toContain('class="pcr-stream-cursor"');
expect(markup).not.toContain(model.text);
});
});
describe("turn strip", () => {
@@ -202,6 +227,32 @@ describe("activity stream", () => {
expect(markup).toContain(`data-testid="activity-turn-${newest}" data-open="true"`);
expect(markup).toContain('data-testid="activity-turn-0" data-open="false"');
});
it("scopes a running turn to evidence at the transcript reveal ceiling", () => {
const artifact = sessionFor("ap-mcp-gate-01");
const turn = artifact.turns.at(-1)!;
const model = artifact.timeline.find(
(entry) => entry.turn === turn.turn && entry.kind === "agent_message",
);
if (model === undefined) throw new Error("staged turn has no model message");
const markup = activity("ap-mcp-gate-01", "all", turn.turn, model.sequence);
expect(markup).toContain(`Turn ${turn.turn} · running`);
const calls = markup.match(
new RegExp(`data-testid="activity-section-${turn.turn}-calls"[\\s\\S]*?</section>`),
)?.[0];
expect(calls).toContain("request_approval");
expect(calls).not.toContain("request_review");
expect(markup).toContain(`data-testid="activity-section-${turn.turn}-authorization"`);
expect(markup).toContain("Authorization (1)");
const header = markup.match(
new RegExp(`data-testid="activity-turn-header-${turn.turn}"[\\s\\S]*?</button>`),
)?.[0];
expect(header).not.toContain("parity");
expect(markup).not.toContain(`data-testid="activity-section-${turn.turn}-diff"`);
expect(markup).not.toContain(`data-testid="activity-section-${turn.turn}-parity"`);
expect(markup).not.toContain('data-testid="activity-session"');
});
});
describe("session header", () => {
@@ -248,6 +299,34 @@ describe("session header", () => {
expect(markup).toContain("Idle · seeded");
expect(markup).not.toContain('data-testid="session-parity-chip"');
});
it("shows and describes why every session control is disabled mid-turn", () => {
const markup = renderToStaticMarkup(
<SessionHeader
entry={entryFor("ap-mcp-gate-01")}
artifact={sessionFor("ap-mcp-gate-01")}
mode="scripted"
codexAvailable={false}
busy
onModeChange={() => undefined}
onReset={() => undefined}
onReplay={() => undefined}
onOpenParity={() => undefined}
/>,
);
expect(markup).toContain('data-testid="running-controls-reason"');
expect(markup).toContain("Controls return when the turn settles.");
for (const testId of [
"chat-mode-scripted",
"chat-mode-codex",
"reset-session",
"replay-session",
]) {
const control = markup.match(new RegExp(`<button[^>]*data-testid="${testId}"[^>]*>`))?.[0];
expect(control, testId).toContain("disabled");
expect(control, testId).toContain('aria-describedby="running-controls-reason"');
}
});
});
describe("chat routes and shell", () => {
@@ -56,6 +56,7 @@ const SECTION_FILTER: Record<SectionName, Phase7ActivityFilter> = {
export function ActivityStream({
artifact,
revealedSequence = Number.MAX_SAFE_INTEGER,
filter,
onFilterChange,
openTurn,
@@ -64,6 +65,8 @@ export function ActivityStream({
showBackToChat,
}: {
artifact: Phase7ChatSessionArtifact;
/** The same inclusive timeline ceiling used by the conversation. */
revealedSequence?: number;
filter: Phase7ActivityFilter;
onFilterChange: (filter: Phase7ActivityFilter) => void;
openTurn: number | null;
@@ -89,20 +92,28 @@ export function ActivityStream({
))}
</div>
{artifact.turns.map((turn) => (
<ActivityTurnGroup
key={turn.turn}
turn={turn}
entries={artifact.timeline.filter((entry) => entry.turn === turn.turn)}
filter={filter}
open={openTurn === null ? turn.turn === newest : openTurn === turn.turn}
onToggle={() => onOpenTurn(turn.turn)}
onBackToChat={onBackToChat}
showBackToChat={showBackToChat}
/>
))}
{artifact.turns.map((turn) => {
const running = turn.lastSequence > revealedSequence;
return (
<ActivityTurnGroup
key={turn.turn}
turn={turn}
entries={artifact.timeline.filter(
(entry) => entry.turn === turn.turn && entry.sequence <= revealedSequence,
)}
running={running}
filter={filter}
open={openTurn === null ? turn.turn === newest : openTurn === turn.turn}
onToggle={() => onOpenTurn(turn.turn)}
onBackToChat={onBackToChat}
showBackToChat={showBackToChat}
/>
);
})}
<SessionGroup artifact={artifact} filter={filter} />
{artifact.turns.some((turn) => turn.lastSequence > revealedSequence) ? null : (
<SessionGroup artifact={artifact} filter={filter} />
)}
</div>
);
}
@@ -110,6 +121,7 @@ export function ActivityStream({
export function ActivityTurnGroup({
turn,
entries,
running,
filter,
open,
onToggle,
@@ -118,6 +130,7 @@ export function ActivityTurnGroup({
}: {
turn: Phase7ChatTurn;
entries: readonly Phase7TimelineEntry[];
running: boolean;
filter: Phase7ActivityFilter;
open: boolean;
onToggle: () => void;
@@ -128,6 +141,29 @@ export function ActivityTurnGroup({
(entry) => entry.kind === "semantic_call" || entry.kind === "semantic_result",
);
const controlPlane = entries.filter((entry) => entry.kind === "control_plane_action");
const authorizationSequences = new Set(
entries.flatMap((entry) =>
entry.kind === "semantic_call" || entry.kind === "semantic_result"
? [entry.authorizationSequence]
: [],
),
);
const authorizationRecords = running
? turn.authorizationRecords.filter((record) => authorizationSequences.has(record.sequence))
: turn.authorizationRecords;
const visibleCounts = running
? {
calls: entries.filter((entry) => entry.kind === "semantic_call").length,
denied: entries.filter(
(entry) =>
entry.kind === "semantic_result" &&
(entry.outcome === "denied" || entry.outcome === "absent"),
).length,
controlPlane: controlPlane.length,
changedDomains: 0,
}
: turn.counts;
const visibleTurn = running ? { ...turn, counts: visibleCounts } : turn;
const visible = (section: SectionName): boolean =>
filter === "all" || SECTION_FILTER[section] === filter;
@@ -147,8 +183,12 @@ export function ActivityTurnGroup({
data-testid={`activity-turn-header-${turn.turn}`}
onClick={onToggle}
>
<span>{turn.turn === 0 ? "Session seed" : `Turn ${turn.turn}`}</span>
<span className="pcr7-muted">{summaryText(turn)}</span>
<span>
{turn.turn === 0 ? "Session seed" : `Turn ${turn.turn}${running ? " · running" : ""}`}
</span>
<span className="pcr7-muted">
{running ? runningSummary(visibleCounts) : summaryText(visibleTurn)}
</span>
</button>
</h3>
{open ? (
@@ -185,13 +225,13 @@ export function ActivityTurnGroup({
</Section>
) : null}
{visible("authorization") && turn.authorizationRecords.length > 0 ? (
{visible("authorization") && authorizationRecords.length > 0 ? (
<Section
name="authorization"
turn={turn.turn}
title={`Authorization (${turn.authorizationRecords.length})`}
title={`Authorization (${authorizationRecords.length})`}
>
<AuthorizationPanel records={turn.authorizationRecords} />
<AuthorizationPanel records={authorizationRecords} />
</Section>
) : null}
@@ -211,13 +251,13 @@ export function ActivityTurnGroup({
</Section>
) : null}
{visible("diff") ? (
{visible("diff") && !running ? (
<Section name="diff" turn={turn.turn} title="State diff">
<TurnDiff turn={turn} />
</Section>
) : null}
{visible("parity") ? (
{visible("parity") && !running ? (
<Section name="parity" turn={turn.turn} title="Parity">
<span className="pcr7-visually-hidden" data-testid={`activity-parity-${turn.turn}`}>
Turn {turn.turn} parity {turn.parity.verdict.replace("_", " ")}
@@ -242,6 +282,11 @@ export function ActivityTurnGroup({
);
}
function runningSummary(counts: Phase7ChatTurn["counts"]): string {
const tools = counts.calls === 1 ? "tool" : "tools";
return `${counts.calls} ${tools}, ${counts.denied} denied, ${counts.controlPlane} control plane`;
}
function Section({
name,
turn,
@@ -91,7 +91,14 @@ function TurnBlock({
data-kind={entry.kind}
data-testid={`chat-entry-${entry.sequence}`}
>
<TimelineEntryBody entry={entry} />
<TimelineEntryBody
entry={entry}
streaming={
!settled &&
entry.kind === "agent_message" &&
entry.sequence === entries.at(-1)?.sequence
}
/>
</li>
))}
</ol>
@@ -51,6 +51,8 @@ export function SessionHeader({
// Controls that would corrupt a running turn are disabled with the reason
// named, never silently inert.
const busyReason = busy ? "A turn is running — wait for it to settle." : undefined;
const runningControlsReasonId = "running-controls-reason";
const codexUnavailableReasonId = "codex-unavailable-reason";
return (
<header ref={headerRef} className="pcr7-session-header" data-testid="session-header" tabIndex={-1}>
@@ -96,6 +98,8 @@ export function SessionHeader({
type="button"
role="radio"
aria-checked={mode === "scripted"}
aria-describedby={busy ? runningControlsReasonId : undefined}
disabled={busy}
data-testid="chat-mode-scripted"
onClick={() => onModeChange("scripted")}
>
@@ -105,7 +109,10 @@ export function SessionHeader({
type="button"
role="radio"
aria-checked={mode === "codex"}
disabled={!codexAvailable}
aria-describedby={
busy ? runningControlsReasonId : !codexAvailable ? codexUnavailableReasonId : undefined
}
disabled={busy || !codexAvailable}
data-testid="chat-mode-codex"
title={
codexAvailable
@@ -118,7 +125,11 @@ export function SessionHeader({
</button>
</div>
{codexAvailable ? null : (
<p className="pcr7-muted" data-testid="chat-codex-unavailable">
<p
id={codexUnavailableReasonId}
className="pcr7-muted"
data-testid="chat-codex-unavailable"
>
Codex mode needs the local provider relay — provider relay not running, see tutorial §4.
Scripted mode drives the same mock control plane offline, and no provider credential
ever reaches this page.
@@ -130,6 +141,7 @@ export function SessionHeader({
variant="ghost"
onClick={onReset}
disabled={busy}
aria-describedby={busy ? runningControlsReasonId : undefined}
title={busyReason}
data-testid="reset-session"
>
@@ -139,6 +151,7 @@ export function SessionHeader({
type="button"
onClick={onReplay}
disabled={busy || (artifact?.remainingScriptedTurns ?? 0) === 0}
aria-describedby={busy ? runningControlsReasonId : undefined}
title={
busyReason ??
((artifact?.remainingScriptedTurns ?? 0) === 0
@@ -149,6 +162,15 @@ export function SessionHeader({
>
Replay scripted session
</Button>
{busy ? (
<p
id={runningControlsReasonId}
className="pcr7-muted"
data-testid="running-controls-reason"
>
Controls return when the turn settles.
</p>
) : null}
</div>
</header>
);
@@ -40,6 +40,9 @@ export type OpenPhase7ChatSession = (
options?: Phase7ChatSessionOptions,
) => Promise<Phase7ChatSessionClient>;
const openLocalSession: OpenPhase7ChatSession = (entry, options) =>
Phase7ChatSession.open(entry, options);
export interface ChatSurfaceProps {
entry: Phase7ScenarioIndexEntry;
/** `replay=fake` plays the whole script on load for screenshot routes. */
@@ -51,6 +54,8 @@ export interface ChatSurfaceProps {
onOpenTurn: (turn: number) => void;
onOpenParity: () => void;
onArtifactChange: (artifact: Phase7ChatSessionArtifact | null) => void;
/** Mirrors the transcript reveal boundary to the activity stream. */
onRevealSequenceChange: (sequence: number) => void;
/**
* Mirrors the settle state to the shell. The chat column is hidden on the
* mobile Activity segment, so a wait target that only lives here would be
@@ -72,9 +77,10 @@ export function ChatSurface({
onOpenTurn,
onOpenParity,
onArtifactChange,
onRevealSequenceChange,
onStateChange,
onCancelScenarioSwitch,
openSession = (nextEntry, options) => Phase7ChatSession.open(nextEntry, options),
openSession = openLocalSession,
headerRef,
}: ChatSurfaceProps) {
const [artifact, setArtifact] = React.useState<Phase7ChatSessionArtifact | null>(null);
@@ -138,11 +144,17 @@ export function ChatSurface({
const target = artifact?.timeline.at(-1)?.sequence ?? 0;
const lastTurn = artifact?.turns.at(-1) ?? null;
const heldModelSequence =
lastTurn === null
? undefined
: artifact?.timeline.find(
(entry) => entry.turn === lastTurn.turn && entry.kind === "agent_message",
)?.sequence;
const holdAt =
lastTurn === null
? 0
: Math.max(
Math.min(lastTurn.firstSequence + 2, lastTurn.lastSequence - 1),
Math.min(heldModelSequence ?? lastTurn.firstSequence, lastTurn.lastSequence - 1),
lastTurn.firstSequence,
);
const ceiling = stageStreaming && lastTurn !== null && lastTurn.turn > 0 ? holdAt : target;
@@ -178,6 +190,10 @@ export function ChatSurface({
onStateChange(chatState);
}, [chatState, onStateChange]);
React.useEffect(() => {
onRevealSequenceChange(revealed);
}, [onRevealSequenceChange, revealed]);
const send = React.useCallback(
async (prompt: string) => {
const session = sessionRef.current;
@@ -48,7 +48,13 @@ export function ScenarioTranscript({
* so a semantic call, a denial, and a control-plane strip look the same
* wherever they are read.
*/
export function TimelineEntryBody({ entry }: { entry: Phase7TimelineEntry }) {
export function TimelineEntryBody({
entry,
streaming = false,
}: {
entry: Phase7TimelineEntry;
streaming?: boolean;
}) {
switch (entry.kind) {
case "user_message":
return (
@@ -57,15 +63,26 @@ export function TimelineEntryBody({ entry }: { entry: Phase7TimelineEntry }) {
</div>
);
case "agent_message":
case "agent_message": {
const text = streaming
? entry.text.slice(0, Math.max(1, Math.ceil(entry.text.length / 2)))
: entry.text;
return (
<div className="pcr7-agent-message">
<div
className="pcr7-agent-message"
data-state={streaming ? "streaming" : "settled"}
data-testid={streaming ? "streaming-model-card" : undefined}
>
<span className="pcr7-gutter" aria-hidden="true">
▸
</span>
<p>{entry.text}</p>
<p>
{text}
{streaming ? <span className="pcr-stream-cursor" aria-hidden="true" /> : null}
</p>
</div>
);
}
case "semantic_call":
return (
Binary file not shown.

Before

Width:  |  Height:  |  Size: 753 KiB

After

Width:  |  Height:  |  Size: 552 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 209 KiB

After

Width:  |  Height:  |  Size: 217 KiB