From 643447b54d3e8bec879257e1a0d91dac9d59174c Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 07:44:09 -0500 Subject: [PATCH] fix(evals): consume search probe response bodies once --- .../internal-artifact-search.ts | 3 ++- .../internal-design-search.ts | 3 ++- .../internal-search-response.test.ts | 24 +++++++++++++++++++ .../internal-search-response.ts | 8 +++++++ 4 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/internal-search-response.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/internal-search-response.ts diff --git a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts index 93e1917bced..7bef494cf69 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-artifact-search.ts @@ -7,6 +7,7 @@ import type { CompanyPortabilityFileEntry } from "../../../packages/shared/src/t import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { makeDefinition } from "./internal-definition.js"; import { mainPerson } from "./internal-privacy.js"; +import { readSearchProbeResponse } from "./internal-search-response.js"; import { checks, entity, hasLink, observation, observeInternal, project, sameTask, type Entity, type InternalObservation } from "./internal-support.js"; const sha = (body: string | Buffer) => createHash("sha256").update(body).digest("hex"); interface HistoricalTask extends Entity { completedAt?: string | null } @@ -118,7 +119,7 @@ function findArtifact(): ChatCatalogCase { async observe(ctx, state): Promise { if (!key) throw new Error("Historical search probe was not created during setup"); const o = await observeInternal(ctx, state); const file = state.finalAttachment as IssueAttachment; const read = async (path: string) => { const url = new URL(path, ctx.input.api.baseURL); if (url.origin !== ctx.input.api.baseURL || !url.pathname.startsWith("/api/")) throw new Error("Historical probe escaped fixture API"); - const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(15000), headers: { Authorization: `Bearer ${key!.token}` } }); const body: unknown = response.ok ? await response.json() : undefined; await response.body?.cancel(); return { status: response.status, body }; }; + const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(15000), headers: { Authorization: `Bearer ${key!.token}` } }); return readSearchProbeResponse(response); }; const [currentSource, attachments, deliveryComments, bytes, search] = await Promise.all([ ctx.input.api.get(`/api/issues/${entity(state, "task").id}`), ctx.input.api.get(`/api/issues/${entity(state, "task").id}/attachments`), ctx.input.api.get(`/api/issues/${entity(state, "task").id}/comments?order=asc`), ctx.input.api.getAttachmentContent(file.id), read(`/api/companies/${o.companyId}/search?q=website`), diff --git a/tests/runner-e2e/chat-catalog-domains/internal-design-search.ts b/tests/runner-e2e/chat-catalog-domains/internal-design-search.ts index ec5c19fae51..87d65b07558 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-design-search.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-design-search.ts @@ -1,6 +1,7 @@ import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { makeDefinition } from "./internal-definition.js"; import { mainPerson } from "./internal-privacy.js"; +import { readSearchProbeResponse } from "./internal-search-response.js"; import { checks, entity, hasLink, observation, observeInternal, project, putDocument, sameTask, task, type Entity, type InternalObservation } from "./internal-support.js"; interface DesignObservation extends InternalObservation { @@ -66,7 +67,7 @@ function findDesign(): ChatCatalogCase { const read = async (path: string) => { const url = new URL(path, ctx.input.api.baseURL); if (url.origin !== ctx.input.api.baseURL || !url.pathname.startsWith("/api/")) throw new Error("Design probe escaped fixture API"); const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key!.token}` } }); - const body: unknown = response.ok ? await response.json() : null; await response.body?.cancel(); return { status: response.status, body }; + return readSearchProbeResponse(response); }; const o = await observeInternal(ctx, state); const hidden = entity(state, "privateTask"); const target = entity(state, "task"); const [privateRead, search] = await Promise.all([read(`/api/issues/${hidden.id}`), read(`/api/companies/${o.companyId}/search?q=launch%20website%20design`)]); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-search-response.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-search-response.test.ts new file mode 100644 index 00000000000..16606737b48 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-search-response.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it, vi } from "vitest"; +import { readSearchProbeResponse } from "./internal-search-response.js"; +describe("actual search response body ownership", () => { + it("reproduces the retained locked-stream observer failure with a real Response", async () => { + const response = new Response(JSON.stringify({ results: [{ id: "source" }] }), { status: 200 }); + await response.json(); expect(response.bodyUsed).toBe(true); expect(response.body?.locked).toBe(true); + await expect(response.body!.cancel()).rejects.toThrow("ReadableStream is locked"); + }); + it("returns actual parsed search bytes without cancelling an already consumed successful body", async () => { + const response = new Response(JSON.stringify({ results: [{ id: "source" }] }), { status: 200 }); + const cancel = vi.spyOn(response.body!, "cancel"); + expect(await readSearchProbeResponse(response)).toEqual({ status: 200, body: { results: [{ id: "source" }] } }); + expect(cancel).not.toHaveBeenCalled(); expect(response.bodyUsed).toBe(true); + }); + it.each([403, 404])("cancels an unread denied body, retaining actual %s and exposing no restricted bytes", async status => { + const response = new Response("Restricted source body", { status }); + const cancel = vi.spyOn(response.body!, "cancel"); + expect(await readSearchProbeResponse(response)).toEqual({ status, body: null }); + expect(cancel).toHaveBeenCalledOnce(); expect(response.bodyUsed).toBe(true); + }); + it("propagates invalid successful JSON instead of inventing empty search results", async () => { + await expect(readSearchProbeResponse(new Response("not JSON", { status: 200 }))).rejects.toThrow(); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-search-response.ts b/tests/runner-e2e/chat-catalog-domains/internal-search-response.ts new file mode 100644 index 00000000000..1d1d6cbd3a5 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-search-response.ts @@ -0,0 +1,8 @@ +/** These two independently authenticated search probes consume a successful + * JSON body once. json() owns that stream reader; cancel() afterward throws + * on its locked stream. An unread denied body still needs cancellation. */ +export async function readSearchProbeResponse(response: Response): Promise<{ status: number; body: unknown }> { + if (response.ok) return { status: response.status, body: await response.json() }; + await response.body?.cancel(); + return { status: response.status, body: null }; +}