From 1422811562e66195c8f830cea68ade19387ea775 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 13:33:08 -0600 Subject: [PATCH] test(evals): keep repeated catalog inventory reads out of the Playwright trace The internal catalog observer re-reads the full company issue list and 1000-row activity on every 1s poll. Playwright recorded each response body, so a failed tasks.organize-open-work attempt (504 seeded tasks) produced ~130MB of trace zips that, sealed twice, exceeded the 256MB private archive allowance and lost all evidence. Route those two exact reads through the existing allowlisted untraced observation path. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/api.test.ts | 13 +++++++++++++ tests/runner-e2e/api.ts | 8 +++++++- .../chat-catalog-domains/internal-support.ts | 4 ++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/api.test.ts b/tests/runner-e2e/api.test.ts index 3fc1e9ea71e..e4e8a90ff5b 100644 --- a/tests/runner-e2e/api.test.ts +++ b/tests/runner-e2e/api.test.ts @@ -83,6 +83,16 @@ it("observes identical native poll results with the actual actor and no traced r expect(fetcher.mock.calls[0]![1]).toMatchObject({ method: "GET", redirect: "error", headers: { Cookie: "synthetic-cookie", Origin: "http://127.0.0.1:3100" } }); expect((fetcher.mock.calls[0]![1] as RequestInit).signal).toBeInstanceOf(AbortSignal); }); +it.each([ + "/api/companies/22222222-2222-4222-8222-222222222222/issues?includeAgentCreatedTasks=true&limit=1000", + "/api/companies/22222222-2222-4222-8222-222222222222/activity?limit=1000", +])("reads the repeated catalog inventory observation %s outside the Playwright trace", async endpoint => { + vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "3100"); + const get = vi.fn(() => { throw new Error("Inventory poll must not enter the Playwright API recorder"); }); + const fetcher = vi.fn(async () => new Response("[]", { status: 200 })); vi.stubGlobal("fetch", fetcher); + await expect(new RunnerApi({ get } as unknown as APIRequestContext).getRunObservation(endpoint)).resolves.toEqual([]); + expect(get).not.toHaveBeenCalled(); expect(String((fetcher.mock.calls[0] as unknown[])[0])).toBe(`http://127.0.0.1:3100${endpoint}`); +}); it("keeps local-trusted run observations cookie-free and preserves typed HTTP failure without retry", async () => { vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "3100"); const api = new RunnerApi({} as APIRequestContext); @@ -97,6 +107,9 @@ it.each([ "/api/companies/22222222-2222-4222-8222-222222222222/agents", "/api/heartbeat-runs/11111111-1111-4111-8111-111111111111?token=anything", "/api/heartbeat-runs/11111111-1111-4111-8111-111111111111/events?afterSeq=0&limit=10&extra=1", + "/api/companies/22222222-2222-4222-8222-222222222222/issues?limit=1000", + "/api/companies/22222222-2222-4222-8222-222222222222/issues?includeAgentCreatedTasks=true&limit=1000&q=x", + "/api/companies/22222222-2222-4222-8222-222222222222/activity?limit=1000&limit=1000", ])("rejects non-observation transport target %s before dispatch", async endpoint => { vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "3100"); const fetcher = vi.fn(); vi.stubGlobal("fetch", fetcher); diff --git a/tests/runner-e2e/api.ts b/tests/runner-e2e/api.ts index 05abf740887..1af470fa9e4 100644 --- a/tests/runner-e2e/api.ts +++ b/tests/runner-e2e/api.ts @@ -102,7 +102,13 @@ export class RunnerApi { const events = new RegExp(`^/api/heartbeat-runs/${uuid}/events$`, "i").test(url.pathname) && [...url.searchParams.keys()].every(key => ["afterSeq", "limit"].includes(key)) && /^\d+$/.test(url.searchParams.get("afterSeq") ?? "") && /^[1-9]\d*$/.test(url.searchParams.get("limit") ?? ""); - if (url.origin !== this.baseURL || url.username || url.password || url.hash || !(companyRuns || run || events)) throw new Error("Run observation requires an allowlisted same-origin GET endpoint"); + // Catalog observers re-read the full company inventory and activity on every poll; tracing those bodies + // made large-fixture failure evidence exceed the private archive allowance, so they share this untraced path. + const exactQuery = (expected: Record) => [...url.searchParams.keys()].length === Object.keys(expected).length + && Object.entries(expected).every(([key, value]) => url.searchParams.getAll(key).length === 1 && url.searchParams.get(key) === value); + const companyIssues = new RegExp(`^/api/companies/${uuid}/issues$`, "i").test(url.pathname) && exactQuery({ includeAgentCreatedTasks: "true", limit: "1000" }); + const companyActivity = new RegExp(`^/api/companies/${uuid}/activity$`, "i").test(url.pathname) && exactQuery({ limit: "1000" }); + if (url.origin !== this.baseURL || url.username || url.password || url.hash || !(companyRuns || run || events || companyIssues || companyActivity)) throw new Error("Run observation requires an allowlisted same-origin GET endpoint"); return this.authenticatedJson("GET", path, undefined, options); } diff --git a/tests/runner-e2e/chat-catalog-domains/internal-support.ts b/tests/runner-e2e/chat-catalog-domains/internal-support.ts index e03c2686394..8de53dcec72 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-support.ts @@ -44,7 +44,7 @@ export async function observeInternal(ctx: ChatCatalogContext, state: Record(`${companyPath(ctx)}/issues?includeAgentCreatedTasks=true&limit=1000`), + api.getRunObservation(`${companyPath(ctx)}/issues?includeAgentCreatedTasks=true&limit=1000`), api.get(`${companyPath(ctx)}/projects?includeArchived=true`), api.get(`${companyPath(ctx)}/goals`), ctx.allRuns(), ]); @@ -53,7 +53,7 @@ export async function observeInternal(ctx: ChatCatalogContext, state: Record row.authorAgentId === fixtures.agent.id && runs.some(run => run.id === row.createdByRunId)); const target = state.task ? entity(state, "task") : undefined; const comments = target ? await api.get(`/api/issues/${target.id}/comments?order=asc`) : []; - const companyActivity = await api.get(`${companyPath(ctx)}/activity?limit=1000`); + const companyActivity = await api.getRunObservation(`${companyPath(ctx)}/activity?limit=1000`); const taskActivity = target ? await api.get(`/api/issues/${target.id}/activity`) : []; const activity = [...new Map([...companyActivity, ...taskActivity].map(row => [row.id, row])).values()]; const renderedLinkEvidence = source ? await collectConversationLinks(ctx.input.page, {