diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index ffe53d181b0..db28e68025a 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -381,6 +381,12 @@ published with permanent campaign dashboards; fixture authors must therefore keep credentials and other private data out of every captured UI state. SVG is active content and is rejected from the packaged evidence entirely. +The standard task and chat evidence collectors read durable run events through the paginated +public API, including completion events beyond the first 1,000 rows. It rejects +missing, repeated, or out-of-order sequence numbers and fails capture after +100 full pages instead of grading a truncated stream. Original incomplete +captures remain failed evidence; qualifying a fix requires a new live attempt. + Every completed local campaign also writes `tests/runner-e2e/results//dashboard.html`. The self-contained page shows the complete profile/environment grid with screenshot thumbnails. diff --git a/tests/runner-e2e/chat-flow.test.ts b/tests/runner-e2e/chat-flow.test.ts index c0a1a174fa8..f69ff74cd3e 100644 --- a/tests/runner-e2e/chat-flow.test.ts +++ b/tests/runner-e2e/chat-flow.test.ts @@ -224,21 +224,34 @@ describe("chat acceptance contracts", () => { ); }); it("retains reset events without requesting a provider log, and does not hide missing real logs", async () => { - const get = vi.fn().mockResolvedValue([{ type: "session_reset" }]); + const get = vi.fn().mockResolvedValue([{ seq: 1, type: "session_reset" }]); const reset = { ...run, resultJson: { conversationReset: true } }; await expect(collectChatRunEvidence({ get }, reset)).resolves.toEqual({ runId: run.id, log: null, - events: [{ type: "session_reset" }], + events: [{ seq: 1, type: "session_reset" }], }); expect(get.mock.calls).toEqual([ - [`/api/heartbeat-runs/${run.id}/events?limit=1000`], + [`/api/heartbeat-runs/${run.id}/events?afterSeq=0&limit=1000`], ]); get.mockRejectedValue(new Error("Run log not found")); await expect(collectChatRunEvidence({ get }, run)).rejects.toThrow( "Run log not found", ); }); + it("retains the tail of a long chat run for invariant checks", async () => { + const firstPage = Array.from({ length: 1000 }, (_, i) => ({ seq: i + 1, eventType: "item.delta" })); + const tail = { seq: 1001, eventType: "run.terminal" }; + const get = vi.fn().mockResolvedValueOnce({ content: "fixture log" }) + .mockResolvedValueOnce(firstPage).mockResolvedValueOnce([tail]); + const evidence = await collectChatRunEvidence({ get }, run); + expect(evidence.events).toEqual([...firstPage, tail]); + expect(get.mock.calls).toEqual([ + [`/api/heartbeat-runs/${run.id}/log?limitBytes=1048576`], + [`/api/heartbeat-runs/${run.id}/events?afterSeq=0&limit=1000`], + [`/api/heartbeat-runs/${run.id}/events?afterSeq=1000&limit=1000`], + ]); + }); it("retains events for an unstarted dependency-blocked wake without asking for a nonexistent log", async () => { const get = vi.fn().mockResolvedValue([]); const suppressed = { ...run, status: "cancelled", errorCode: "issue_dependencies_blocked", startedAt: null }; diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts index db36a37124c..a77712a674d 100644 --- a/tests/runner-e2e/chat-flow.ts +++ b/tests/runner-e2e/chat-flow.ts @@ -7,6 +7,7 @@ import type { import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; import { isBlockedUnstartedWake } from "./non-execution-wake.js"; +import { collectRunEvents } from "./run-observations.js"; import { chatMarker } from "./chat-cases.js"; import { assertChatRememberedAfterRestart, assertChatStartupStopped, isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js"; import { enableChatThroughSettings, runChatInterruption, runChatSettingsLifecycle } from "./chat-stories.js"; @@ -239,7 +240,9 @@ export async function collectChatRunEvidence( log: isResetRun(run) || isBlockedUnstartedWake({ ...run }) ? null : await api.get(`/api/heartbeat-runs/${run.id}/log?limitBytes=1048576`), - events: await api.get(`/api/heartbeat-runs/${run.id}/events?limit=1000`), + events: await collectRunEvents((afterSeq, limit) => + api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`), + ), }; } diff --git a/tests/runner-e2e/everyday.test.ts b/tests/runner-e2e/everyday.test.ts index 5016adf2688..c1322eb7943 100644 --- a/tests/runner-e2e/everyday.test.ts +++ b/tests/runner-e2e/everyday.test.ts @@ -212,7 +212,7 @@ describe("manual everyday workflow catalog", () => { const selected = selectRunnerExecutions( parseRunnerSelectors(["--suite", "everyday-workflows"]), ); - expect(selected).toHaveLength(38); + expect(selected).toHaveLength(47); expect(selected.every((e) => e.profile.generation === "native")).toBe(true); expect( selectRunnerExecutions( diff --git a/tests/runner-e2e/run-observations.ts b/tests/runner-e2e/run-observations.ts index e0f18d49cef..1e5b1949e22 100644 --- a/tests/runner-e2e/run-observations.ts +++ b/tests/runner-e2e/run-observations.ts @@ -1,3 +1,28 @@ +/** Read the complete durable stream; never grade a silently truncated page. */ +export async function collectRunEvents( + loadPage: (afterSeq: number, limit: number) => Promise, +): Promise { + const pageSize = 1000; + const events: T[] = []; + let afterSeq = 0; + for (let pageNumber = 0; pageNumber < 100; pageNumber += 1) { + const page = await loadPage(afterSeq, pageSize); + if (!Array.isArray(page) || page.length > pageSize) { + throw new Error("Run event evidence returned an invalid page"); + } + for (const event of page) { + const seq = event?.seq; + if (!Number.isSafeInteger(seq) || seq <= afterSeq) { + throw new Error("Run event evidence has a missing or non-increasing sequence"); + } + afterSeq = seq; + events.push(event as T); + } + if (page.length < pageSize) return events; + } + throw new Error("Run event evidence exceeded 100 pages; refusing incomplete evidence"); +} + export interface ObservableRunState { status?: string | null; errorCode?: string | null; diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 9649bf1da52..5e8392c529a 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -21,6 +21,7 @@ import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js"; import { evaluateMatcher, persistedFinalRunMessage, type MatcherResult } from "./matchers.js"; import { acceptedPlanSessionResetFailures, + collectRunEvents, hasTerminalMalformedPlanConfirmation, isControlPlaneGovernedResponseWait, isNonExecutingReviewFenceRun, @@ -696,8 +697,8 @@ for (const execution of executions) { ), ), events: await capture(() => - api.get( - `/api/heartbeat-runs/${candidate.id}/events?limit=1000`, + collectRunEvents((afterSeq, limit) => + api.get(`/api/heartbeat-runs/${candidate.id}/events?afterSeq=${afterSeq}&limit=${limit}`), ), ), })), @@ -1694,8 +1695,8 @@ for (const execution of executions) { try { return { runId: candidate.id, - events: await api.get( - `/api/heartbeat-runs/${candidate.id}/events?limit=1000`, + events: await collectRunEvents((afterSeq, limit) => + api.get(`/api/heartbeat-runs/${candidate.id}/events?afterSeq=${afterSeq}&limit=${limit}`), ), error: null, }; diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.ts index e7c4cb95fff..9c71367fa0b 100644 --- a/tests/runner-e2e/support.test.ts +++ b/tests/runner-e2e/support.test.ts @@ -38,6 +38,7 @@ import { } from "./ports.js"; import { acceptedPlanSessionResetFailures, + collectRunEvents, hasTerminalMalformedPlanConfirmation, isControlPlaneGovernedResponseWait, isNonExecutingReviewFenceRun, @@ -48,6 +49,47 @@ import { import { runnerE2EWebServerCommand } from "./web-server-command.js"; const cleanupDirectories: string[] = []; + +describe("complete run event evidence", () => { + const page = Array.from({ length: 1000 }, (_, i) => ({ seq: i + 1, eventType: "item.delta" })); + it("reads completion events beyond the first 1000 rows", async () => { + const terminal = ["run.result.proposed", "run.result.accepted", "run.terminal"] + .map((eventType, i) => ({ seq: 1001 + i, eventType })); + const load = vi.fn().mockResolvedValueOnce(page).mockResolvedValueOnce(terminal); + const events = await collectRunEvents(load); + expect(events).toEqual([...page, ...terminal]); + expect(load.mock.calls).toEqual([[0, 1000], [1000, 1000]]); + }); + it("checks for another page even at an exact page boundary", async () => { + const load = vi.fn().mockResolvedValueOnce(page).mockResolvedValueOnce([]); + expect(await collectRunEvents(load)).toEqual(page); + expect(load).toHaveBeenCalledTimes(2); + }); + it.each([ + null, {}, [{ eventType: "run.terminal" }], [{ seq: 0 }], [{ seq: -1 }], + [{ seq: 1.5 }], [{ seq: "1" }], [{ seq: NaN }], [{ seq: Infinity }], + [{ seq: 2 }, { seq: 1 }], [{ seq: 1 }, { seq: 1 }], [...page, { seq: 1001 }], + ])("rejects malformed evidence page %#", async (malformed) => { + await expect(collectRunEvents(async () => malformed)).rejects.toThrow("Run event evidence"); + }); + it("rejects a repeated cursor instead of accepting duplicate events", async () => { + const load = vi.fn().mockResolvedValue(page); + await expect(collectRunEvents(load)).rejects.toThrow("non-increasing sequence"); + expect(load).toHaveBeenCalledTimes(2); + }); + it("propagates a missing later page without returning partial evidence", async () => { + const load = vi.fn().mockResolvedValueOnce(page).mockRejectedValueOnce(new Error("Unavailable")); + await expect(collectRunEvents(load)).rejects.toThrow("Unavailable"); + }); + it("fails closed when a stream never ends within the bounded capture", async () => { + const load = vi.fn(async (afterSeq: number) => page.map((event) => ({ ...event, seq: event.seq + afterSeq }))); + await expect(collectRunEvents(load)).rejects.toThrow("refusing incomplete evidence"); + expect(load).toHaveBeenCalledTimes(100); + }); +}); + + + afterEach(async () => { vi.unstubAllEnvs(); vi.unstubAllGlobals();