diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts index a5544ad6f87..a8c3277c0cb 100644 --- a/tests/runner-e2e/chat-flow.ts +++ b/tests/runner-e2e/chat-flow.ts @@ -8,7 +8,7 @@ import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; import { isBlockedUnstartedWake } from "./non-execution-wake.js"; import { chatMarker } from "./chat-cases.js"; -import { isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js"; +import { assertChatStartupStopped, isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js"; // Public API observations only: this driver never fabricates provider results or writes DB state. export interface ChatIssue { @@ -504,6 +504,12 @@ export async function runChatFlow(input: ChatFlowInput) { ).toEqual( oldComments.filter((c) => c.createdByRunId === cancelledId), ); + if (caseId === "stop-startup-new-resume" && cancelledId) { + const stopped = await api.get(`/api/heartbeat-runs/${cancelledId}`); + const events = await api.get>(`/api/heartbeat-runs/${cancelledId}/events?limit=1000`); + await input.evidence("chat-startup-stop-result.json", { stopped, events }); + assertChatStartupStopped(stopped, events); + } await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 }); await expect( page.getByText("New session", { exact: true }), diff --git a/tests/runner-e2e/chat-hardening.test.ts b/tests/runner-e2e/chat-hardening.test.ts index 9e54a75e44c..7721870b3db 100644 --- a/tests/runner-e2e/chat-hardening.test.ts +++ b/tests/runner-e2e/chat-hardening.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { assertChatHire, assertChatSourceReview, assertCommittedSendRetry, assertGroundedChatStatus, isChatStopReady } from "./chat-hardening.js"; +import { assertChatHire, assertChatSourceReview, assertChatStartupStopped, assertCommittedSendRetry, assertGroundedChatStatus, isChatStopReady } from "./chat-hardening.js"; import { runnerMatrix } from "./catalog.js"; import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; @@ -67,6 +67,15 @@ describe("agent chat hardening oracles", () => { expect(() => assertCommittedSendRetry({ ...valid, runs: [] })).toThrow(); }); + it("requires dispatched Stop with no late turn and reports a missed startup checkpoint separately", () => { + const cancellation = { scope: "run", dispatchState: "acknowledged", dispatched: true, recordedAt: "2026-09-21T12:00:00Z" }; + const stopped = { ...run, status: "cancelled", resultJson: { nativeCancellation: cancellation } }; + expect(() => assertChatStartupStopped(stopped, [])).not.toThrow(); + expect(() => assertChatStartupStopped({ ...stopped, resultJson: { nativeCancellation: { ...cancellation, dispatched: false } } }, [])).toThrow(); + expect(() => assertChatStartupStopped(stopped, [{ eventType: "turn.started", createdAt: "2026-09-21T12:00:01Z" }])).toThrow(/must never submit/); + expect(() => assertChatStartupStopped(stopped, [{ eventType: "turn.started", createdAt: "2026-09-21T11:59:59Z" }])).toThrow(/Harness missed startup Stop boundary/); + }); + it("keeps the paid hardening matrix explicit and limits API-tool opt-in to coordination", () => { const cells = runnerMatrix.filter(cell => cell.suite.id === "agent-chat-hardening"); expect(cells).toHaveLength(18); diff --git a/tests/runner-e2e/chat-hardening.ts b/tests/runner-e2e/chat-hardening.ts index 74eebb42ae0..09f5a379caf 100644 --- a/tests/runner-e2e/chat-hardening.ts +++ b/tests/runner-e2e/chat-hardening.ts @@ -76,6 +76,17 @@ export function isChatStopReady(events: Array<{ eventType?: unknown }>, phase: " return phase === "active" ? active : !active && events.some(event => event.eventType === "native.process_start_requested"); } +export function assertChatStartupStopped(run: ChatRun, events: Array<{ eventType?: unknown; createdAt?: string }>) { + const cancellation = run.resultJson?.nativeCancellation as Record | undefined; + const started = events.filter(event => ["turn.started", "turn.accepted"].includes(String(event.eventType))); + if (started.some(event => Date.parse(event.createdAt ?? "") < Date.parse(String(cancellation?.recordedAt)))) { + throw new Error("Harness missed startup Stop boundary: provider turn started before the Stop intent"); + } + expect(cancellation).toMatchObject({ scope: "run", dispatchState: "acknowledged", dispatched: true }); + expect(run.status).toBe("cancelled"); + expect(started, "A stopped startup must never submit a provider turn").toHaveLength(0); +} + export async function runChatHardeningFlow(context: { input: ChatFlowInput; marker: string; issue(): ChatIssue; turn(message: string, count: number): Promise; idle(count: number): Promise; @@ -91,6 +102,7 @@ export async function runChatHardeningFlow(context: { const latestReply = async () => (await comments()).filter(comment => comment.authorAgentId === f.agent.id).at(-1)?.body ?? ""; const output = (id: string, token: string) => readChatOutputDocument(api, id, token); + try { if (execution.task.id === "hire-delegate-reuse") { const hireName = `Morgan Reviewer ${nonce}`; await turn(`Hire exactly one teammate named ${hireName}, reporting to you, using your native runner, model, and available AI connection. Have that teammate write a concise launch checklist as a saved Paperclip document containing ${marker}. Put the work in one assigned task in ${project.name}. Link it here and let the teammate complete it.`, 2); @@ -197,4 +209,27 @@ export async function runChatHardeningFlow(context: { await page.unroute("**/api/issues/*/comments", handler); } } else throw new Error(`Unsupported chat hardening case ${execution.task.id}`); + } finally { + // Preserve the records the matchers inspected even when an assertion fails. + // Failed API reads remain explicit evidence instead of replacing the failure. + const capture = async (name: string, load: () => Promise) => { + try { return { name, value: await load() }; } + catch (error) { return { name, error: String(error) }; } + }; + const state = await Promise.all([ + capture("agents", agents), + capture("runs", allRuns), + capture("chatComments", comments), + capture("tasks", async () => Promise.all((await tasks()).map(async task => ({ + task, + comments: await capture("comments", () => api.get(`/api/issues/${task.id}/comments?order=asc`)), + documents: await capture("documents", async () => { + const summaries = await api.get>(`/api/issues/${task.id}/documents`); + return Promise.all(summaries.map(document => capture(document.key, + () => api.get(`/api/issues/${task.id}/documents/${encodeURIComponent(document.key)}`)))); + }), + })))), + ]); + await input.evidence("chat-hardening-state.json", state); + } }