test: retain chat failure evidence and verify startup Stop dispatch

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-flow.ts

@@ -8,7 +8,7 @@ import type { LiveFixtureValues } from "./live-fixtures.js";
 import type { MatrixExecution } from "./types.js";
 import { isBlockedUnstartedWake } from "./non-execution-wake.js";
 import { chatMarker } from "./chat-cases.js";
-import { isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js";
+import { assertChatStartupStopped, isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js";
 
 // Public API observations only: this driver never fabricates provider results or writes DB state.
 export interface ChatIssue {
@@ -504,6 +504,12 @@ export async function runChatFlow(input: ChatFlowInput) {
           ).toEqual(
             oldComments.filter((c) => c.createdByRunId === cancelledId),
           );
+        if (caseId === "stop-startup-new-resume" && cancelledId) {
+          const stopped = await api.get<ChatRun>(`/api/heartbeat-runs/${cancelledId}`);
+          const events = await api.get<Array<{ eventType?: unknown; createdAt?: string }>>(`/api/heartbeat-runs/${cancelledId}/events?limit=1000`);
+          await input.evidence("chat-startup-stop-result.json", { stopped, events });
+          assertChatStartupStopped(stopped, events);
+        }
         await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
         await expect(
           page.getByText("New session", { exact: true }),

tests/runner-e2e/chat-hardening.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { assertChatHire, assertChatSourceReview, assertCommittedSendRetry, assertGroundedChatStatus, isChatStopReady } from "./chat-hardening.js";
+import { assertChatHire, assertChatSourceReview, assertChatStartupStopped, assertCommittedSendRetry, assertGroundedChatStatus, isChatStopReady } from "./chat-hardening.js";
 import { runnerMatrix } from "./catalog.js";
 import { buildRunnerE2EProcessEnvironment } from "./harness-env.js";
 
@@ -67,6 +67,15 @@ describe("agent chat hardening oracles", () => {
     expect(() => assertCommittedSendRetry({ ...valid, runs: [] })).toThrow();
   });
 
+  it("requires dispatched Stop with no late turn and reports a missed startup checkpoint separately", () => {
+    const cancellation = { scope: "run", dispatchState: "acknowledged", dispatched: true, recordedAt: "2026-09-21T12:00:00Z" };
+    const stopped = { ...run, status: "cancelled", resultJson: { nativeCancellation: cancellation } };
+    expect(() => assertChatStartupStopped(stopped, [])).not.toThrow();
+    expect(() => assertChatStartupStopped({ ...stopped, resultJson: { nativeCancellation: { ...cancellation, dispatched: false } } }, [])).toThrow();
+    expect(() => assertChatStartupStopped(stopped, [{ eventType: "turn.started", createdAt: "2026-09-21T12:00:01Z" }])).toThrow(/must never submit/);
+    expect(() => assertChatStartupStopped(stopped, [{ eventType: "turn.started", createdAt: "2026-09-21T11:59:59Z" }])).toThrow(/Harness missed startup Stop boundary/);
+  });
+
   it("keeps the paid hardening matrix explicit and limits API-tool opt-in to coordination", () => {
     const cells = runnerMatrix.filter(cell => cell.suite.id === "agent-chat-hardening");
     expect(cells).toHaveLength(18);

tests/runner-e2e/chat-hardening.ts

@@ -76,6 +76,17 @@ export function isChatStopReady(events: Array<{ eventType?: unknown }>, phase: "
   return phase === "active" ? active : !active && events.some(event => event.eventType === "native.process_start_requested");
 }
 
+export function assertChatStartupStopped(run: ChatRun, events: Array<{ eventType?: unknown; createdAt?: string }>) {
+  const cancellation = run.resultJson?.nativeCancellation as Record<string, unknown> | undefined;
+  const started = events.filter(event => ["turn.started", "turn.accepted"].includes(String(event.eventType)));
+  if (started.some(event => Date.parse(event.createdAt ?? "") < Date.parse(String(cancellation?.recordedAt)))) {
+    throw new Error("Harness missed startup Stop boundary: provider turn started before the Stop intent");
+  }
+  expect(cancellation).toMatchObject({ scope: "run", dispatchState: "acknowledged", dispatched: true });
+  expect(run.status).toBe("cancelled");
+  expect(started, "A stopped startup must never submit a provider turn").toHaveLength(0);
+}
+
 export async function runChatHardeningFlow(context: {
   input: ChatFlowInput; marker: string; issue(): ChatIssue;
   turn(message: string, count: number): Promise<void>; idle(count: number): Promise<void>;
@@ -91,6 +102,7 @@ export async function runChatHardeningFlow(context: {
   const latestReply = async () => (await comments()).filter(comment => comment.authorAgentId === f.agent.id).at(-1)?.body ?? "";
   const output = (id: string, token: string) => readChatOutputDocument(api, id, token);
 
+  try {
   if (execution.task.id === "hire-delegate-reuse") {
     const hireName = `Morgan Reviewer ${nonce}`;
     await turn(`Hire exactly one teammate named ${hireName}, reporting to you, using your native runner, model, and available AI connection. Have that teammate write a concise launch checklist as a saved Paperclip document containing ${marker}. Put the work in one assigned task in ${project.name}. Link it here and let the teammate complete it.`, 2);
@@ -197,4 +209,27 @@ export async function runChatHardeningFlow(context: {
       await page.unroute("**/api/issues/*/comments", handler);
     }
   } else throw new Error(`Unsupported chat hardening case ${execution.task.id}`);
+  } finally {
+    // Preserve the records the matchers inspected even when an assertion fails.
+    // Failed API reads remain explicit evidence instead of replacing the failure.
+    const capture = async (name: string, load: () => Promise<unknown>) => {
+      try { return { name, value: await load() }; }
+      catch (error) { return { name, error: String(error) }; }
+    };
+    const state = await Promise.all([
+      capture("agents", agents),
+      capture("runs", allRuns),
+      capture("chatComments", comments),
+      capture("tasks", async () => Promise.all((await tasks()).map(async task => ({
+        task,
+        comments: await capture("comments", () => api.get(`/api/issues/${task.id}/comments?order=asc`)),
+        documents: await capture("documents", async () => {
+          const summaries = await api.get<Array<{ key: string }>>(`/api/issues/${task.id}/documents`);
+          return Promise.all(summaries.map(document => capture(document.key,
+            () => api.get(`/api/issues/${task.id}/documents/${encodeURIComponent(document.key)}`))));
+        }),
+      })))),
+    ]);
+    await input.evidence("chat-hardening-state.json", state);
+  }
 }