From ecc6752b6ae932163aba8e3d5c98c23c7513d75c Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 27 Sep 2026 18:19:59 -0500 Subject: [PATCH] test(e2e): prove saved API response pagination through EOF --- tests/runner-e2e/api-response-reading.test.ts | 80 ++++++++++++++++-- tests/runner-e2e/api-response-reading.ts | 83 +++++++++++++++++-- tests/runner-e2e/runner.spec.ts | 10 ++- 3 files changed, 156 insertions(+), 17 deletions(-) diff --git a/tests/runner-e2e/api-response-reading.test.ts b/tests/runner-e2e/api-response-reading.test.ts index 77bdc2d49bc..0ccfb053edc 100644 --- a/tests/runner-e2e/api-response-reading.test.ts +++ b/tests/runner-e2e/api-response-reading.test.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { describe, expect, it, vi } from "vitest"; import type { RunnerApi } from "./api.js"; -import { apiResponseReadingTask, readResponseProof, responseEvidenceCode, responseEvidenceDescription, successfulApiReadCount } from "./api-response-reading.js"; +import { apiResponseReadingTask, gradeApiResponsePaging, readResponseProof, responseEvidenceCode, responseEvidenceDescription } from "./api-response-reading.js"; describe("large API response evidence fixture", () => { it("keeps evidence beyond both preview and inline limits and out of the assignment", () => { @@ -12,12 +12,6 @@ describe("large API response evidence fixture", () => { expect(apiResponseReadingTask.buildPrompt("fixture")).not.toContain(code); expect(apiResponseReadingTask.buildPrompt("fixture")).toContain("responseText.nextOffsetBytes"); }); - it("counts only real completed API tool events, never narrative claims or missing evidence", () => { - const event = { eventType: "tool.execution.completed", payload: { prpEvent: { payload: { name: "call_api", status: "completed" } } } }; - expect(successfulApiReadCount([event])).toBe(1); - expect(successfulApiReadCount([])).toBe(0); - expect(successfulApiReadCount([{ ...event, eventType: "item.delta" }, { ...event, payload: null }, { ...event, payload: { prpEvent: { payload: { name: "call_api", status: "failed" } } } }])).toBe(0); - }); it("downloads exact run-attributed proof bytes rather than trusting the agent's report", async () => { const bytes = Buffer.from("independent evidence\n"); const proof = { id: "proof", issueId: "issue", originatingRunId: "run", originalFilename: "api-response-proof.txt", byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }; @@ -34,3 +28,75 @@ describe("large API response evidence fixture", () => { await expect(readResponseProof(api, "issue", "run")).rejects.toThrow("disagrees"); }); }); + +describe("saved API response pagination oracle", () => { + const sourceIssueId = "source-issue"; + const assetId = "source-asset"; + const bytes = Buffer.from(JSON.stringify({ id: sourceIssueId, description: "synthetic padding ".repeat(2000) })); + const operation = "GET /api/assets/{assetId}/content"; + const event = (eventType: string, payload: unknown) => ({ eventType, payload: { prpEvent: { payload } } }); + const call = (id: string, input: unknown, result: unknown, status = "completed") => [ + event("item.started", { item: { id, type: "tool_use", name: "call_api", input } }), + event("item.completed", { item: { id, type: "tool_result", tool_use_id: id, result } }), + event("tool.execution.completed", { name: "call_api", executionId: id, status }), + ]; + const source = () => call("source-read", { operationId: "GET /api/issues/{id}", pathParams: { id: sourceIssueId } }, { + ok: true, status: 200, apiOperationId: "GET /api/issues/{id}", artifact: { + artifactId: assetId, byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex"), + url: `/api/assets/${assetId}/content`, + }, + }); + const page = (offset: number, id = `page-${offset}`, artifactId = assetId) => { + const end = Math.min(offset + 8192, bytes.length); + return call(id, { operationId: operation, pathParams: { assetId: artifactId }, responseText: { offsetBytes: offset, limitBytes: 8192 } }, { + ok: true, status: 200, apiOperationId: operation, data: bytes.subarray(offset, end).toString(), + responseText: { offsetBytes: offset, totalBytes: bytes.length, nextOffsetBytes: end === bytes.length ? null : end }, + }); + }; + const pages = () => Array.from({ length: Math.ceil(bytes.length / 8192) }, (_, i) => page(i * 8192)).flat(); + it("accepts correlated source artifact pages through EOF, including a harmless identical retry", () => { + expect(gradeApiResponsePaging([...source(), ...page(0, "retry"), ...pages()], sourceIssueId).passed).toBe(true); + }); + it("rejects unrelated completed calls", () => { + const unrelated = [1, 2, 3].flatMap(i => call(`unrelated-${i}`, { operationId: "GET /api/agents/me" }, { ok: true, status: 200 })); + expect(gradeApiResponsePaging(unrelated, sourceIssueId).passed).toBe(false); + }); + it("rejects repeating the first page without reaching EOF", () => { + expect(gradeApiResponsePaging([...source(), ...page(0), ...page(0, "again"), ...page(0, "again-2")], sourceIssueId).passed).toBe(false); + }); + it("rejects a gap even when the last page claims EOF", () => { + expect(gradeApiResponsePaging([...source(), ...pages().filter((_e, i) => i < 3 || i >= 6)], sourceIssueId).passed).toBe(false); + }); + it("rejects reading a different artifact", () => { + const other = Array.from({ length: Math.ceil(bytes.length / 8192) }, (_, i) => page(i * 8192, `other-${i}`, "other-asset")).flat(); + expect(gradeApiResponsePaging([...source(), ...other], sourceIssueId).passed).toBe(false); + }); + it("requires the exact seeded source issue", () => { + expect(gradeApiResponsePaging([...source(), ...pages()], "other-issue").passed).toBe(false); + }); + it("requires a matched successful completion for every page", () => { + const incomplete = pages().filter(e => e.eventType !== "tool.execution.completed"); + expect(gradeApiResponsePaging([...source(), ...incomplete], sourceIssueId).passed).toBe(false); + }); + it.each([ + ["requested offset", { offsetBytes: 1, totalBytes: bytes.length, nextOffsetBytes: 8192 }], + ["next offset", { offsetBytes: 0, totalBytes: bytes.length, nextOffsetBytes: 8193 }], + ["total size", { offsetBytes: 0, totalBytes: bytes.length + 1, nextOffsetBytes: 8192 }], + ["premature EOF", { offsetBytes: 0, totalBytes: bytes.length, nextOffsetBytes: null }], + ])("rejects mismatched %s metadata", (_label, responseText) => { + const malformed = call("bad-page", { + operationId: operation, pathParams: { assetId }, responseText: { offsetBytes: 0, limitBytes: 8192 }, + }, { ok: true, status: 206, apiOperationId: operation, data: bytes.subarray(0, 8192).toString(), responseText }); + expect(gradeApiResponsePaging([...source(), ...pages(), ...malformed], sourceIssueId).passed).toBe(false); + }); + it("rejects contradictory duplicate bytes and a wrong source artifact digest", () => { + const corrupt = call("corrupt-page", { + operationId: operation, pathParams: { assetId }, responseText: { offsetBytes: 0, limitBytes: 8192 }, + }, { + ok: true, status: 206, apiOperationId: operation, data: "x".repeat(8192), + responseText: { offsetBytes: 0, totalBytes: bytes.length, nextOffsetBytes: 8192 }, + }); + expect(gradeApiResponsePaging([...source(), ...pages(), ...corrupt], sourceIssueId).passed).toBe(false); + expect(gradeApiResponsePaging([...source(), ...corrupt, ...pages().slice(3)], sourceIssueId).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/api-response-reading.ts b/tests/runner-e2e/api-response-reading.ts index a167182c0f5..b91db1c71ab 100644 --- a/tests/runner-e2e/api-response-reading.ts +++ b/tests/runner-e2e/api-response-reading.ts @@ -27,12 +27,83 @@ export function responseEvidenceDescription(nonce: string) { return `${"Synthetic diagnostic padding.\n".repeat(1400)}\nEvidence code: ${responseEvidenceCode(nonce)}\n`; } -export function successfulApiReadCount(events: readonly { eventType?: string; payload?: unknown }[]) { - return events.filter(event => { - const outer = event.payload as { prpEvent?: { payload?: { name?: string; status?: string } } } | null; - const payload = outer?.prpEvent?.payload; - return event.eventType === "tool.execution.completed" && payload?.name === "call_api" && payload.status === "completed"; - }).length; +/** Correlate trusted PRP call inputs, results, and completion receipts. */ +export function gradeApiResponsePaging(events: readonly { eventType?: string; payload?: unknown }[], sourceIssueId: string) { + const object = (value: unknown): Record => + value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + const inputs = new Map>(); + const results = new Map>(); + const completed = new Set(); + for (const event of events) { + const payload = object(object(object(event.payload).prpEvent).payload); + const item = object(payload.item); + if (event.eventType === "item.started" && item.type === "tool_use" && item.name === "call_api" && typeof item.id === "string") { + inputs.set(item.id, object(item.input)); + } + if (event.eventType === "item.completed" && item.type === "tool_result" && typeof item.id === "string" && + (item.tool_use_id === undefined || item.tool_use_id === item.id)) results.set(item.id, object(item.result)); + if (event.eventType === "tool.execution.completed" && payload.name === "call_api" && payload.status === "completed" && typeof payload.executionId === "string") { + completed.add(payload.executionId); + } + } + const calls = [...inputs].flatMap(([id, input]) => { + const result = results.get(id); + return completed.has(id) && result?.ok === true && [200, 206].includes(Number(result.status)) ? [{ input, result }] : []; + }); + const sourceOperation = "GET /api/issues/{id}"; + const assetOperation = "GET /api/assets/{assetId}/content"; + const sources = calls.filter(call => call.input.operationId === sourceOperation && call.result.apiOperationId === sourceOperation && + object(call.input.pathParams).id === sourceIssueId); + let failure = "No successful call_api source read with a saved response artifact"; + for (const source of sources) { + const artifact = object(source.result.artifact); + const assetId = artifact.artifactId; + const total = artifact.byteSize; + if (typeof assetId !== "string" || typeof total !== "number" || !Number.isSafeInteger(total) || total <= 24 * 1024 || total > 1024 * 1024 || + typeof artifact.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(artifact.sha256) || artifact.url !== `/api/assets/${assetId}/content`) continue; + const pages = new Map(); + let malformed = false; + for (const call of calls) { + if (call.input.operationId !== assetOperation || object(call.input.pathParams).assetId !== assetId) continue; + const requested = object(call.input.responseText); + const returned = object(call.result.responseText); + const offset = requested.offsetBytes; + const limit = requested.limitBytes; + const next = returned.nextOffsetBytes; + const data = call.result.data; + if (call.result.apiOperationId !== assetOperation || typeof offset !== "number" || !Number.isSafeInteger(offset) || offset < 0 || offset >= total || + typeof limit !== "number" || !Number.isSafeInteger(limit) || limit < 1 || limit > 8192 || returned.offsetBytes !== offset || returned.totalBytes !== total || + typeof data !== "string") { malformed = true; break; } + const bytes = Buffer.from(data, "utf8"); + const end = offset + bytes.length; + if (bytes.length === 0 || bytes.length > limit || end > total || (end === total ? next !== null : next !== end)) { + malformed = true; break; + } + const prior = pages.get(offset); + if (prior && (!prior.bytes.equals(bytes) || prior.next !== next)) { malformed = true; break; } + pages.set(offset, { bytes, next: next as number | null }); + } + if (malformed) { failure = "Artifact paging request, returned offsets, bytes, or EOF disagree"; continue; } + const chunks: Buffer[] = []; + const offsets: number[] = []; + let offset = 0; + let eofReached = false; + while (pages.has(offset)) { + const page = pages.get(offset)!; + chunks.push(page.bytes); offsets.push(offset); + if (page.next === null) { eofReached = true; break; } + offset = page.next; + } + if (!eofReached || offsets.length < 2) { failure = "Saved artifact pages do not cover offset zero through EOF without gaps"; continue; } + const bytes = Buffer.concat(chunks); + const digest = createHash("sha256").update(bytes).digest("hex"); + if (bytes.length !== total || digest !== artifact.sha256) { failure = "Paged bytes disagree with the source response artifact digest"; continue; } + try { + if (object(JSON.parse(bytes.toString("utf8"))).id !== sourceIssueId) { failure = "Paged artifact belongs to another source issue"; continue; } + } catch { failure = "Paged source response is not complete JSON"; continue; } + return { passed: true, sourceIssueId, artifactId: assetId, totalBytes: total, pageCount: offsets.length, offsets, eofReached, sha256: digest }; + } + return { passed: false, sourceIssueId, failure }; } export const apiResponseReadingTask: RunnerTaskFixture = { diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 6771566eb52..d97661f4a43 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,5 +1,4 @@ -import { readResponseProof, responseEvidenceDescription, successfulApiReadCount } from "./api-response-reading.js"; - +import { gradeApiResponsePaging, readResponseProof, responseEvidenceDescription } from "./api-response-reading.js"; import { runInstructionPersistenceFlow } from "./instruction-persistence.js"; import { observeBrowserBootstrap } from "./browser-bootstrap-diagnostics.js"; import { runAccountingFlow } from "./accounting-flow.js"; @@ -537,6 +536,7 @@ for (const execution of executions) { const marker = execution.task.buildVisibleMarker(nonce); const title = execution.task.buildTitle(nonce); let prompt = execution.task.buildPrompt(nonce); + let apiResponseSourceId: string | undefined; let lifecycleBlockerId: string | null = null; const credentials = credentialValues(); const secrets = normalizedSecrets(Object.values(credentials)); @@ -803,6 +803,7 @@ for (const execution of executions) { title: `Synthetic diagnostic evidence ${nonce}`, description: responseEvidenceDescription(nonce), status: "backlog", }); + apiResponseSourceId = source.id; prompt = prompt.replaceAll("{{API_RESPONSE_SOURCE_ID}}", source.id); } @@ -1908,8 +1909,9 @@ for (const execution of executions) { } if (execution.suite.id === "api-response-reading") { - const completedReads = successfulApiReadCount(runEventsByRun.flatMap(captured => captured.events)); - if (completedReads < 3) invariantFailures.push(`Expected source read and multiple bounded artifact reads; observed ${completedReads} completed API calls`); + const paging = gradeApiResponsePaging(runEventsByRun.flatMap(captured => captured.events), apiResponseSourceId ?? ""); + await writeSanitizedJson(snapshotsDir, "api-response-pagination.json", paging, secrets); + if (!paging.passed) invariantFailures.push(`Bounded response paging was not proven: ${paging.failure}`); } const context = record(run.contextSnapshot);