test(e2e): prove saved API response pagination through EOF

tests/runner-e2e/api-response-reading.test.ts

@@ -1,7 +1,7 @@
 import { createHash } from "node:crypto";
 import { describe, expect, it, vi } from "vitest";
 import type { RunnerApi } from "./api.js";
-import { apiResponseReadingTask, readResponseProof, responseEvidenceCode, responseEvidenceDescription, successfulApiReadCount } from "./api-response-reading.js";
+import { apiResponseReadingTask, gradeApiResponsePaging, readResponseProof, responseEvidenceCode, responseEvidenceDescription } from "./api-response-reading.js";
 
 describe("large API response evidence fixture", () => {
   it("keeps evidence beyond both preview and inline limits and out of the assignment", () => {
@@ -12,12 +12,6 @@ describe("large API response evidence fixture", () => {
     expect(apiResponseReadingTask.buildPrompt("fixture")).not.toContain(code);
     expect(apiResponseReadingTask.buildPrompt("fixture")).toContain("responseText.nextOffsetBytes");
   });
-  it("counts only real completed API tool events, never narrative claims or missing evidence", () => {
-    const event = { eventType: "tool.execution.completed", payload: { prpEvent: { payload: { name: "call_api", status: "completed" } } } };
-    expect(successfulApiReadCount([event])).toBe(1);
-    expect(successfulApiReadCount([])).toBe(0);
-    expect(successfulApiReadCount([{ ...event, eventType: "item.delta" }, { ...event, payload: null }, { ...event, payload: { prpEvent: { payload: { name: "call_api", status: "failed" } } } }])).toBe(0);
-  });
   it("downloads exact run-attributed proof bytes rather than trusting the agent's report", async () => {
     const bytes = Buffer.from("independent evidence\n");
     const proof = { id: "proof", issueId: "issue", originatingRunId: "run", originalFilename: "api-response-proof.txt", byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") };
@@ -34,3 +28,75 @@ describe("large API response evidence fixture", () => {
     await expect(readResponseProof(api, "issue", "run")).rejects.toThrow("disagrees");
   });
 });
+
+describe("saved API response pagination oracle", () => {
+  const sourceIssueId = "source-issue";
+  const assetId = "source-asset";
+  const bytes = Buffer.from(JSON.stringify({ id: sourceIssueId, description: "synthetic padding ".repeat(2000) }));
+  const operation = "GET /api/assets/{assetId}/content";
+  const event = (eventType: string, payload: unknown) => ({ eventType, payload: { prpEvent: { payload } } });
+  const call = (id: string, input: unknown, result: unknown, status = "completed") => [
+    event("item.started", { item: { id, type: "tool_use", name: "call_api", input } }),
+    event("item.completed", { item: { id, type: "tool_result", tool_use_id: id, result } }),
+    event("tool.execution.completed", { name: "call_api", executionId: id, status }),
+  ];
+  const source = () => call("source-read", { operationId: "GET /api/issues/{id}", pathParams: { id: sourceIssueId } }, {
+    ok: true, status: 200, apiOperationId: "GET /api/issues/{id}", artifact: {
+      artifactId: assetId, byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex"),
+      url: `/api/assets/${assetId}/content`,
+    },
+  });
+  const page = (offset: number, id = `page-${offset}`, artifactId = assetId) => {
+    const end = Math.min(offset + 8192, bytes.length);
+    return call(id, { operationId: operation, pathParams: { assetId: artifactId }, responseText: { offsetBytes: offset, limitBytes: 8192 } }, {
+      ok: true, status: 200, apiOperationId: operation, data: bytes.subarray(offset, end).toString(),
+      responseText: { offsetBytes: offset, totalBytes: bytes.length, nextOffsetBytes: end === bytes.length ? null : end },
+    });
+  };
+  const pages = () => Array.from({ length: Math.ceil(bytes.length / 8192) }, (_, i) => page(i * 8192)).flat();
+  it("accepts correlated source artifact pages through EOF, including a harmless identical retry", () => {
+    expect(gradeApiResponsePaging([...source(), ...page(0, "retry"), ...pages()], sourceIssueId).passed).toBe(true);
+  });
+  it("rejects unrelated completed calls", () => {
+    const unrelated = [1, 2, 3].flatMap(i => call(`unrelated-${i}`, { operationId: "GET /api/agents/me" }, { ok: true, status: 200 }));
+    expect(gradeApiResponsePaging(unrelated, sourceIssueId).passed).toBe(false);
+  });
+  it("rejects repeating the first page without reaching EOF", () => {
+    expect(gradeApiResponsePaging([...source(), ...page(0), ...page(0, "again"), ...page(0, "again-2")], sourceIssueId).passed).toBe(false);
+  });
+  it("rejects a gap even when the last page claims EOF", () => {
+    expect(gradeApiResponsePaging([...source(), ...pages().filter((_e, i) => i < 3 || i >= 6)], sourceIssueId).passed).toBe(false);
+  });
+  it("rejects reading a different artifact", () => {
+    const other = Array.from({ length: Math.ceil(bytes.length / 8192) }, (_, i) => page(i * 8192, `other-${i}`, "other-asset")).flat();
+    expect(gradeApiResponsePaging([...source(), ...other], sourceIssueId).passed).toBe(false);
+  });
+  it("requires the exact seeded source issue", () => {
+    expect(gradeApiResponsePaging([...source(), ...pages()], "other-issue").passed).toBe(false);
+  });
+  it("requires a matched successful completion for every page", () => {
+    const incomplete = pages().filter(e => e.eventType !== "tool.execution.completed");
+    expect(gradeApiResponsePaging([...source(), ...incomplete], sourceIssueId).passed).toBe(false);
+  });
+  it.each([
+    ["requested offset", { offsetBytes: 1, totalBytes: bytes.length, nextOffsetBytes: 8192 }],
+    ["next offset", { offsetBytes: 0, totalBytes: bytes.length, nextOffsetBytes: 8193 }],
+    ["total size", { offsetBytes: 0, totalBytes: bytes.length + 1, nextOffsetBytes: 8192 }],
+    ["premature EOF", { offsetBytes: 0, totalBytes: bytes.length, nextOffsetBytes: null }],
+  ])("rejects mismatched %s metadata", (_label, responseText) => {
+    const malformed = call("bad-page", {
+      operationId: operation, pathParams: { assetId }, responseText: { offsetBytes: 0, limitBytes: 8192 },
+    }, { ok: true, status: 206, apiOperationId: operation, data: bytes.subarray(0, 8192).toString(), responseText });
+    expect(gradeApiResponsePaging([...source(), ...pages(), ...malformed], sourceIssueId).passed).toBe(false);
+  });
+  it("rejects contradictory duplicate bytes and a wrong source artifact digest", () => {
+    const corrupt = call("corrupt-page", {
+      operationId: operation, pathParams: { assetId }, responseText: { offsetBytes: 0, limitBytes: 8192 },
+    }, {
+      ok: true, status: 206, apiOperationId: operation, data: "x".repeat(8192),
+      responseText: { offsetBytes: 0, totalBytes: bytes.length, nextOffsetBytes: 8192 },
+    });
+    expect(gradeApiResponsePaging([...source(), ...pages(), ...corrupt], sourceIssueId).passed).toBe(false);
+    expect(gradeApiResponsePaging([...source(), ...corrupt, ...pages().slice(3)], sourceIssueId).passed).toBe(false);
+  });
+});

tests/runner-e2e/api-response-reading.ts

@@ -27,12 +27,83 @@ export function responseEvidenceDescription(nonce: string) {
   return `${"Synthetic diagnostic padding.\n".repeat(1400)}\nEvidence code: ${responseEvidenceCode(nonce)}\n`;
 }
 
-export function successfulApiReadCount(events: readonly { eventType?: string; payload?: unknown }[]) {
-  return events.filter(event => {
-    const outer = event.payload as { prpEvent?: { payload?: { name?: string; status?: string } } } | null;
-    const payload = outer?.prpEvent?.payload;
-    return event.eventType === "tool.execution.completed" && payload?.name === "call_api" && payload.status === "completed";
-  }).length;
+/** Correlate trusted PRP call inputs, results, and completion receipts. */
+export function gradeApiResponsePaging(events: readonly { eventType?: string; payload?: unknown }[], sourceIssueId: string) {
+  const object = (value: unknown): Record<string, unknown> =>
+    value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
+  const inputs = new Map<string, Record<string, unknown>>();
+  const results = new Map<string, Record<string, unknown>>();
+  const completed = new Set<string>();
+  for (const event of events) {
+    const payload = object(object(object(event.payload).prpEvent).payload);
+    const item = object(payload.item);
+    if (event.eventType === "item.started" && item.type === "tool_use" && item.name === "call_api" && typeof item.id === "string") {
+      inputs.set(item.id, object(item.input));
+    }
+    if (event.eventType === "item.completed" && item.type === "tool_result" && typeof item.id === "string" &&
+      (item.tool_use_id === undefined || item.tool_use_id === item.id)) results.set(item.id, object(item.result));
+    if (event.eventType === "tool.execution.completed" && payload.name === "call_api" && payload.status === "completed" && typeof payload.executionId === "string") {
+      completed.add(payload.executionId);
+    }
+  }
+  const calls = [...inputs].flatMap(([id, input]) => {
+    const result = results.get(id);
+    return completed.has(id) && result?.ok === true && [200, 206].includes(Number(result.status)) ? [{ input, result }] : [];
+  });
+  const sourceOperation = "GET /api/issues/{id}";
+  const assetOperation = "GET /api/assets/{assetId}/content";
+  const sources = calls.filter(call => call.input.operationId === sourceOperation && call.result.apiOperationId === sourceOperation &&
+    object(call.input.pathParams).id === sourceIssueId);
+  let failure = "No successful call_api source read with a saved response artifact";
+  for (const source of sources) {
+    const artifact = object(source.result.artifact);
+    const assetId = artifact.artifactId;
+    const total = artifact.byteSize;
+    if (typeof assetId !== "string" || typeof total !== "number" || !Number.isSafeInteger(total) || total <= 24 * 1024 || total > 1024 * 1024 ||
+      typeof artifact.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(artifact.sha256) || artifact.url !== `/api/assets/${assetId}/content`) continue;
+    const pages = new Map<number, { bytes: Buffer; next: number | null }>();
+    let malformed = false;
+    for (const call of calls) {
+      if (call.input.operationId !== assetOperation || object(call.input.pathParams).assetId !== assetId) continue;
+      const requested = object(call.input.responseText);
+      const returned = object(call.result.responseText);
+      const offset = requested.offsetBytes;
+      const limit = requested.limitBytes;
+      const next = returned.nextOffsetBytes;
+      const data = call.result.data;
+      if (call.result.apiOperationId !== assetOperation || typeof offset !== "number" || !Number.isSafeInteger(offset) || offset < 0 || offset >= total ||
+        typeof limit !== "number" || !Number.isSafeInteger(limit) || limit < 1 || limit > 8192 || returned.offsetBytes !== offset || returned.totalBytes !== total ||
+        typeof data !== "string") { malformed = true; break; }
+      const bytes = Buffer.from(data, "utf8");
+      const end = offset + bytes.length;
+      if (bytes.length === 0 || bytes.length > limit || end > total || (end === total ? next !== null : next !== end)) {
+        malformed = true; break;
+      }
+      const prior = pages.get(offset);
+      if (prior && (!prior.bytes.equals(bytes) || prior.next !== next)) { malformed = true; break; }
+      pages.set(offset, { bytes, next: next as number | null });
+    }
+    if (malformed) { failure = "Artifact paging request, returned offsets, bytes, or EOF disagree"; continue; }
+    const chunks: Buffer[] = [];
+    const offsets: number[] = [];
+    let offset = 0;
+    let eofReached = false;
+    while (pages.has(offset)) {
+      const page = pages.get(offset)!;
+      chunks.push(page.bytes); offsets.push(offset);
+      if (page.next === null) { eofReached = true; break; }
+      offset = page.next;
+    }
+    if (!eofReached || offsets.length < 2) { failure = "Saved artifact pages do not cover offset zero through EOF without gaps"; continue; }
+    const bytes = Buffer.concat(chunks);
+    const digest = createHash("sha256").update(bytes).digest("hex");
+    if (bytes.length !== total || digest !== artifact.sha256) { failure = "Paged bytes disagree with the source response artifact digest"; continue; }
+    try {
+      if (object(JSON.parse(bytes.toString("utf8"))).id !== sourceIssueId) { failure = "Paged artifact belongs to another source issue"; continue; }
+    } catch { failure = "Paged source response is not complete JSON"; continue; }
+    return { passed: true, sourceIssueId, artifactId: assetId, totalBytes: total, pageCount: offsets.length, offsets, eofReached, sha256: digest };
+  }
+  return { passed: false, sourceIssueId, failure };
 }
 
 export const apiResponseReadingTask: RunnerTaskFixture = {

tests/runner-e2e/runner.spec.ts

@@ -1,5 +1,4 @@
-import { readResponseProof, responseEvidenceDescription, successfulApiReadCount } from "./api-response-reading.js";
-
+import { gradeApiResponsePaging, readResponseProof, responseEvidenceDescription } from "./api-response-reading.js";
 import { runInstructionPersistenceFlow } from "./instruction-persistence.js";
 import { observeBrowserBootstrap } from "./browser-bootstrap-diagnostics.js";
 import { runAccountingFlow } from "./accounting-flow.js";
@@ -537,6 +536,7 @@ for (const execution of executions) {
     const marker = execution.task.buildVisibleMarker(nonce);
     const title = execution.task.buildTitle(nonce);
     let prompt = execution.task.buildPrompt(nonce);
+    let apiResponseSourceId: string | undefined;
     let lifecycleBlockerId: string | null = null;
     const credentials = credentialValues();
     const secrets = normalizedSecrets(Object.values(credentials));
@@ -803,6 +803,7 @@ for (const execution of executions) {
           title: `Synthetic diagnostic evidence ${nonce}`,
           description: responseEvidenceDescription(nonce), status: "backlog",
         });
+        apiResponseSourceId = source.id;
         prompt = prompt.replaceAll("{{API_RESPONSE_SOURCE_ID}}", source.id);
       }
 
@@ -1908,8 +1909,9 @@ for (const execution of executions) {
       }
 
       if (execution.suite.id === "api-response-reading") {
-        const completedReads = successfulApiReadCount(runEventsByRun.flatMap(captured => captured.events));
-        if (completedReads < 3) invariantFailures.push(`Expected source read and multiple bounded artifact reads; observed ${completedReads} completed API calls`);
+        const paging = gradeApiResponsePaging(runEventsByRun.flatMap(captured => captured.events), apiResponseSourceId ?? "");
+        await writeSanitizedJson(snapshotsDir, "api-response-pagination.json", paging, secrets);
+        if (!paging.passed) invariantFailures.push(`Bounded response paging was not proven: ${paging.failure}`);
       }
 
       const context = record(run.contextSnapshot);