diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index c45e00c2981..4ffcb521fe8 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -2833,8 +2833,12 @@ Each ordinary live attempt collects company-wide saved feedback after retiring runs and before fixture teardown, even when cancellation or grading fails. This includes feedback from child and background runs. Full paginated bodies and native tool invocation outcomes are checkpointed in the access-controlled -`snapshots/agent-feedback.json` artifact. Collection has a 30-second deadline; -missing detail pages, truncated run lists, active work, and unavailable APIs +`snapshots/agent-feedback.json` artifact. Collection has a 30-second deadline. +Event reads use the remaining shared read budget, with up to five seconds +reserved for the final checkpoint. Other reads keep a five-second request cap +so a stalled earlier endpoint leaves time to inspect run events. All reads +remain bounded by the shared read cutoff and are not retried. +Missing detail pages, truncated run lists, active work, and unavailable APIs remain explicit partial/unavailable observations. Legacy shell helpers do not have complete native invocation coverage. An empty saved-feedback list does not prove that a tool was available, and a completed tool receipt does not by itself diff --git a/tests/runner-e2e/feedback.test.ts b/tests/runner-e2e/feedback.test.ts index dce4d204e8e..1f1d82ead62 100644 --- a/tests/runner-e2e/feedback.test.ts +++ b/tests/runner-e2e/feedback.test.ts @@ -435,3 +435,93 @@ it("bounds raw event inspection while retaining the observed feedback receipt pr expect(summary.invocations.unknown).toBeLessThan(100); expect(retained!.diagnostics).toContain("evidence_limit_reached"); }); + + +it("allows a slow event read to finish within the shared collection deadline", async () => { + let now = 0, eventReads = 0; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + let retained: FeedbackEvidence | undefined; + const slowApi = { get: async (url: string, options: { timeout: number }) => { + if (url.includes("agent-commentary?")) return { items: [], total: 0 }; + if (url === "/api/companies/company/heartbeat-runs?limit=1000") return [{ id: "run", status: "succeeded" }]; + if (url === "/api/heartbeat-runs/run") return { id: "run", status: "succeeded", runtimeMode: "native" }; + if (url.includes("/events?")) { + eventReads++; + now += Math.min(8_000, options.timeout); + if (options.timeout < 8_000) throw new Error("request timeout"); + return [event(1, "tool.execution.completed", { executionId: "call", name: "submit_suggestion", status: "completed" })]; + } + throw new Error("Unexpected read"); + } } as Pick; + try { + const summary = await collectAttemptFeedback({ companyId: "company", deadlineAt: 30_000, api: slowApi, + persist: async evidence => { retained = structuredClone(evidence); }, + }); + expect(summary).toMatchObject({ collection: "complete", invocationCollection: "complete", runsInspected: 1, + invocations: { completed: 1, failed: 0, unknown: 0 } }); + expect(retained!.runCoverage).toEqual([{ runId: "run", classification: "native_events" }]); + expect(retained!.diagnostics).toEqual([]); + expect(eventReads).toBe(1); + expect(now).toBe(8_000); + } finally { clock.mockRestore(); } +}); + +it("still times out a stalled event read and reserves time to persist saved feedback", async () => { + let now = 0, eventReads = 0; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + let retained: FeedbackEvidence | undefined; + const stalledApi = { get: async (url: string, options: { timeout: number }) => { + if (url.includes("agent-commentary?")) return { items: [entry("saved")], total: 1 }; + if (url.includes("agent-commentary/")) return entry("saved"); + if (url === "/api/companies/company/heartbeat-runs?limit=1000") return [{ id: "run", status: "succeeded" }]; + if (url === "/api/heartbeat-runs/run") return { id: "run", status: "succeeded", runtimeMode: "native" }; + if (url.includes("/events?")) { + eventReads++; + now += options.timeout; + throw new Error("request timeout: secret details"); + } + throw new Error("Unexpected read"); + } } as Pick; + try { + const summary = await collectAttemptFeedback({ companyId: "company", deadlineAt: 30_000, api: stalledApi, + persist: async evidence => { retained = structuredClone(evidence); }, + }); + expect(now).toBe(25_000); + expect(summary).toMatchObject({ collection: "complete", invocationCollection: "partial", bodiesCollected: 1, runsInspected: 0 }); + expect(retained!.entries).toEqual([entry("saved")]); + expect(retained!.diagnostics).toEqual(["events_read_failed"]); + expect(JSON.stringify(retained)).not.toContain("secret details"); + expect(eventReads).toBe(1); + } finally { clock.mockRestore(); } +}); + + +it("keeps time for event coverage after an earlier feedback endpoint stalls", async () => { + let now = 0, eventReads = 0; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + let retained: FeedbackEvidence | undefined; + const stalledListApi = { get: async (url: string, options: { timeout: number }) => { + if (url.includes("agent-commentary?")) { + now += options.timeout; + throw new Error("request timeout"); + } + if (url === "/api/companies/company/heartbeat-runs?limit=1000") return [{ id: "run", status: "succeeded" }]; + if (url === "/api/heartbeat-runs/run") return { id: "run", status: "succeeded", runtimeMode: "native" }; + if (url.includes("/events?")) { + eventReads++; + now += Math.min(8_000, options.timeout); + if (options.timeout < 8_000) throw new Error("request timeout"); + return []; + } + throw new Error("Unexpected read"); + } } as Pick; + try { + const summary = await collectAttemptFeedback({ companyId: "company", deadlineAt: 30_000, api: stalledListApi, + persist: async evidence => { retained = structuredClone(evidence); }, + }); + expect(summary).toMatchObject({ collection: "unavailable", invocationCollection: "complete", runsInspected: 1 }); + expect(retained!.diagnostics).toEqual(["feedback_read_failed"]); + expect(eventReads).toBe(1); + expect(now).toBe(13_000); + } finally { clock.mockRestore(); } +}); diff --git a/tests/runner-e2e/feedback.ts b/tests/runner-e2e/feedback.ts index f75d57988eb..c0c3ec22971 100644 --- a/tests/runner-e2e/feedback.ts +++ b/tests/runner-e2e/feedback.ts @@ -137,10 +137,10 @@ export async function collectAttemptFeedback(input: { const evidence: FeedbackEvidence = { version: 1, companyId: input.companyId, entries: [], invocations: [], runCoverage, diagnostics: [], summary }; let retainedBytes = 0, checkpointBytes = 0, eventBytes = 0, eventCount = 0; const diagnose = (code: FeedbackEvidence["diagnostics"][number]) => { if (!evidence.diagnostics.includes(code)) evidence.diagnostics.push(code); }; - const get = (url: string) => { + const get = (url: string, options?: { useRemainingBudget: boolean }) => { const remaining = readDeadlineAt - Date.now(); if (remaining <= 0) { diagnose("deadline_reached"); throw new Error("Feedback collection deadline reached"); } - return input.api.get(url, { timeout: Math.min(5_000, remaining) }); + return input.api.get(url, { timeout: options?.useRemainingBudget ? remaining : Math.min(5_000, remaining) }); }; const persist = async (force = false) => { summary.bodiesCollected = evidence.entries.length; @@ -252,7 +252,7 @@ export async function collectAttemptFeedback(input: { let eventsComplete = false; try { for (let pageNumber = 0; pageNumber < 100; pageNumber++) { - const page = await get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=100`); + const page = await get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=100`, { useRemainingBudget: true }); if (!Array.isArray(page) || page.length > 100) throw new Error("Invalid events"); for (const event of page) { if (!Number.isSafeInteger(event.seq) || Number(event.seq) <= afterSeq) throw new Error("Invalid event sequence");