fix(evals): capture complete durable run event streams

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/README.md

@@ -381,6 +381,12 @@ published with permanent campaign dashboards; fixture authors must therefore
 keep credentials and other private data out of every captured UI state. SVG is
 active content and is rejected from the packaged evidence entirely.
 
+The standard task and chat evidence collectors read durable run events through the paginated
+public API, including completion events beyond the first 1,000 rows. It rejects
+missing, repeated, or out-of-order sequence numbers and fails capture after
+100 full pages instead of grading a truncated stream. Original incomplete
+captures remain failed evidence; qualifying a fix requires a new live attempt.
+
 Every completed local campaign also writes
 `tests/runner-e2e/results/<campaign>/dashboard.html`. The self-contained page
 shows the complete profile/environment grid with screenshot thumbnails.

tests/runner-e2e/chat-flow.test.ts

@@ -224,21 +224,34 @@ describe("chat acceptance contracts", () => {
     );
   });
   it("retains reset events without requesting a provider log, and does not hide missing real logs", async () => {
-    const get = vi.fn().mockResolvedValue([{ type: "session_reset" }]);
+    const get = vi.fn().mockResolvedValue([{ seq: 1, type: "session_reset" }]);
     const reset = { ...run, resultJson: { conversationReset: true } };
     await expect(collectChatRunEvidence({ get }, reset)).resolves.toEqual({
       runId: run.id,
       log: null,
-      events: [{ type: "session_reset" }],
+      events: [{ seq: 1, type: "session_reset" }],
     });
     expect(get.mock.calls).toEqual([
-      [`/api/heartbeat-runs/${run.id}/events?limit=1000`],
+      [`/api/heartbeat-runs/${run.id}/events?afterSeq=0&limit=1000`],
     ]);
     get.mockRejectedValue(new Error("Run log not found"));
     await expect(collectChatRunEvidence({ get }, run)).rejects.toThrow(
       "Run log not found",
     );
   });
+  it("retains the tail of a long chat run for invariant checks", async () => {
+    const firstPage = Array.from({ length: 1000 }, (_, i) => ({ seq: i + 1, eventType: "item.delta" }));
+    const tail = { seq: 1001, eventType: "run.terminal" };
+    const get = vi.fn().mockResolvedValueOnce({ content: "fixture log" })
+      .mockResolvedValueOnce(firstPage).mockResolvedValueOnce([tail]);
+    const evidence = await collectChatRunEvidence({ get }, run);
+    expect(evidence.events).toEqual([...firstPage, tail]);
+    expect(get.mock.calls).toEqual([
+      [`/api/heartbeat-runs/${run.id}/log?limitBytes=1048576`],
+      [`/api/heartbeat-runs/${run.id}/events?afterSeq=0&limit=1000`],
+      [`/api/heartbeat-runs/${run.id}/events?afterSeq=1000&limit=1000`],
+    ]);
+  });
   it("retains events for an unstarted dependency-blocked wake without asking for a nonexistent log", async () => {
     const get = vi.fn().mockResolvedValue([]);
     const suppressed = { ...run, status: "cancelled", errorCode: "issue_dependencies_blocked", startedAt: null };

tests/runner-e2e/chat-flow.ts

@@ -7,6 +7,7 @@ import type {
 import type { LiveFixtureValues } from "./live-fixtures.js";
 import type { MatrixExecution } from "./types.js";
 import { isBlockedUnstartedWake } from "./non-execution-wake.js";
+import { collectRunEvents } from "./run-observations.js";
 import { chatMarker } from "./chat-cases.js";
 import { assertChatRememberedAfterRestart, assertChatStartupStopped, isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js";
 import { enableChatThroughSettings, runChatInterruption, runChatSettingsLifecycle } from "./chat-stories.js";
@@ -239,7 +240,9 @@ export async function collectChatRunEvidence(
     log: isResetRun(run) || isBlockedUnstartedWake({ ...run })
       ? null
       : await api.get(`/api/heartbeat-runs/${run.id}/log?limitBytes=1048576`),
-    events: await api.get(`/api/heartbeat-runs/${run.id}/events?limit=1000`),
+    events: await collectRunEvents((afterSeq, limit) =>
+      api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`),
+    ),
   };
 }
 

tests/runner-e2e/everyday.test.ts

@@ -212,7 +212,7 @@ describe("manual everyday workflow catalog", () => {
     const selected = selectRunnerExecutions(
       parseRunnerSelectors(["--suite", "everyday-workflows"]),
     );
-    expect(selected).toHaveLength(38);
+    expect(selected).toHaveLength(47);
     expect(selected.every((e) => e.profile.generation === "native")).toBe(true);
     expect(
       selectRunnerExecutions(

tests/runner-e2e/run-observations.ts

@@ -1,3 +1,28 @@
+/** Read the complete durable stream; never grade a silently truncated page. */
+export async function collectRunEvents<T extends { seq?: number }>(
+  loadPage: (afterSeq: number, limit: number) => Promise<unknown>,
+): Promise<T[]> {
+  const pageSize = 1000;
+  const events: T[] = [];
+  let afterSeq = 0;
+  for (let pageNumber = 0; pageNumber < 100; pageNumber += 1) {
+    const page = await loadPage(afterSeq, pageSize);
+    if (!Array.isArray(page) || page.length > pageSize) {
+      throw new Error("Run event evidence returned an invalid page");
+    }
+    for (const event of page) {
+      const seq = event?.seq;
+      if (!Number.isSafeInteger(seq) || seq <= afterSeq) {
+        throw new Error("Run event evidence has a missing or non-increasing sequence");
+      }
+      afterSeq = seq;
+      events.push(event as T);
+    }
+    if (page.length < pageSize) return events;
+  }
+  throw new Error("Run event evidence exceeded 100 pages; refusing incomplete evidence");
+}
+
 export interface ObservableRunState {
   status?: string | null;
   errorCode?: string | null;

tests/runner-e2e/runner.spec.ts

@@ -21,6 +21,7 @@ import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js";
 import { evaluateMatcher, persistedFinalRunMessage, type MatcherResult } from "./matchers.js";
 import {
   acceptedPlanSessionResetFailures,
+  collectRunEvents,
   hasTerminalMalformedPlanConfirmation,
   isControlPlaneGovernedResponseWait,
   isNonExecutingReviewFenceRun,
@@ -697,8 +698,8 @@ for (const execution of executions) {
             ),
           ),
           events: await capture(() =>
-            api.get<RunEventRecord[]>(
-              `/api/heartbeat-runs/${candidate.id}/events?limit=1000`,
+            collectRunEvents<RunEventRecord>((afterSeq, limit) =>
+              api.get(`/api/heartbeat-runs/${candidate.id}/events?afterSeq=${afterSeq}&limit=${limit}`),
             ),
           ),
         })),
@@ -1695,8 +1696,8 @@ for (const execution of executions) {
             try {
               return {
                 runId: candidate.id,
-                events: await api.get<RunEventRecord[]>(
-                  `/api/heartbeat-runs/${candidate.id}/events?limit=1000`,
+                events: await collectRunEvents<RunEventRecord>((afterSeq, limit) =>
+                  api.get(`/api/heartbeat-runs/${candidate.id}/events?afterSeq=${afterSeq}&limit=${limit}`),
                 ),
                 error: null,
               };

tests/runner-e2e/support.test.ts

@@ -38,6 +38,7 @@ import {
 } from "./ports.js";
 import {
   acceptedPlanSessionResetFailures,
+  collectRunEvents,
   hasTerminalMalformedPlanConfirmation,
   isControlPlaneGovernedResponseWait,
   isNonExecutingReviewFenceRun,
@@ -68,6 +69,47 @@ it("rejects reasoning mislabeled as assistant text without echoing private conte
   assistant.payload.prpEvent.payload.update.kind = "agentMessage";
   expect(reasoningProjectionFailures([assistant])).toEqual([]);
 });
+
+describe("complete run event evidence", () => {
+  const page = Array.from({ length: 1000 }, (_, i) => ({ seq: i + 1, eventType: "item.delta" }));
+  it("reads completion events beyond the first 1000 rows", async () => {
+    const terminal = ["run.result.proposed", "run.result.accepted", "run.terminal"]
+      .map((eventType, i) => ({ seq: 1001 + i, eventType }));
+    const load = vi.fn().mockResolvedValueOnce(page).mockResolvedValueOnce(terminal);
+    const events = await collectRunEvents(load);
+    expect(events).toEqual([...page, ...terminal]);
+    expect(load.mock.calls).toEqual([[0, 1000], [1000, 1000]]);
+  });
+  it("checks for another page even at an exact page boundary", async () => {
+    const load = vi.fn().mockResolvedValueOnce(page).mockResolvedValueOnce([]);
+    expect(await collectRunEvents(load)).toEqual(page);
+    expect(load).toHaveBeenCalledTimes(2);
+  });
+  it.each([
+    null, {}, [{ eventType: "run.terminal" }], [{ seq: 0 }], [{ seq: -1 }],
+    [{ seq: 1.5 }], [{ seq: "1" }], [{ seq: NaN }], [{ seq: Infinity }],
+    [{ seq: 2 }, { seq: 1 }], [{ seq: 1 }, { seq: 1 }], [...page, { seq: 1001 }],
+  ])("rejects malformed evidence page %#", async (malformed) => {
+    await expect(collectRunEvents(async () => malformed)).rejects.toThrow("Run event evidence");
+  });
+  it("rejects a repeated cursor instead of accepting duplicate events", async () => {
+    const load = vi.fn().mockResolvedValue(page);
+    await expect(collectRunEvents(load)).rejects.toThrow("non-increasing sequence");
+    expect(load).toHaveBeenCalledTimes(2);
+  });
+  it("propagates a missing later page without returning partial evidence", async () => {
+    const load = vi.fn().mockResolvedValueOnce(page).mockRejectedValueOnce(new Error("Unavailable"));
+    await expect(collectRunEvents(load)).rejects.toThrow("Unavailable");
+  });
+  it("fails closed when a stream never ends within the bounded capture", async () => {
+    const load = vi.fn(async (afterSeq: number) => page.map((event) => ({ ...event, seq: event.seq + afterSeq })));
+    await expect(collectRunEvents(load)).rejects.toThrow("refusing incomplete evidence");
+    expect(load).toHaveBeenCalledTimes(100);
+  });
+});
+
+
+
 afterEach(async () => {
   vi.unstubAllEnvs();
   vi.unstubAllGlobals();