PaperClipAI/tests/runner-e2e/api-response-reading.test.ts

import { createHash } from "node:crypto";
import { describe, expect, it, vi } from "vitest";
import type { RunnerApi } from "./api.js";
import { apiResponseReadingTask, gradeApiResponsePaging, readResponseProof, responseEvidenceCode, responseEvidenceDescription } from "./api-response-reading.js";

describe("large API response evidence fixture", () => {
  it("keeps evidence beyond both preview and inline limits and out of the assignment", () => {
    const source = responseEvidenceDescription("fixture");
    const code = responseEvidenceCode("fixture");
    expect(Buffer.byteLength(source)).toBeGreaterThan(24 * 1024);
    expect(source.indexOf(code)).toBeGreaterThan(24 * 1024);
    expect(apiResponseReadingTask.buildPrompt("fixture")).not.toContain(code);
    expect(apiResponseReadingTask.buildPrompt("fixture")).toContain("responseText.nextOffsetBytes");
  });
  it("downloads exact run-attributed proof bytes rather than trusting the agent's report", async () => {
    const bytes = Buffer.from("independent evidence\n");
    const proof = { id: "proof", issueId: "issue", originatingRunId: "run", originalFilename: "api-response-proof.txt", byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") };
    const download = vi.fn().mockResolvedValue({ ok: () => true, body: async () => bytes });
    const get = vi.fn().mockResolvedValue([proof]);
    const api = { get, request: { get: download } } as unknown as RunnerApi;
    expect((await readResponseProof(api, "issue", "run")).content).toBe(bytes.toString());
    expect(download).toHaveBeenCalledWith("/api/attachments/proof/content?download=1");
    get.mockResolvedValue([{ ...proof, originatingRunId: "other" }]);
    await expect(readResponseProof(api, "issue", "run")).rejects.toThrow("observed 0");
    get.mockResolvedValue([proof, proof]);
    await expect(readResponseProof(api, "issue", "run")).rejects.toThrow("observed 2");
    get.mockResolvedValue([{ ...proof, sha256: "mismatch" }]);
    await expect(readResponseProof(api, "issue", "run")).rejects.toThrow("disagrees");
  });
});

describe("saved API response pagination oracle", () => {
  const sourceIssueId = "source-issue";
  const assetId = "source-asset";
  const bytes = Buffer.from(JSON.stringify({ id: sourceIssueId, description: "synthetic padding ".repeat(2000) }));
  const operation = "GET /api/assets/{assetId}/content";
  const event = (eventType: string, payload: unknown) => ({ eventType, payload: { prpEvent: { payload } } });
  const call = (id: string, input: unknown, result: unknown, status = "completed") => [
    event("item.started", { item: { id, type: "tool_use", name: "call_api", input } }),
    event("item.completed", { item: { id, type: "tool_result", tool_use_id: id, result } }),
    event("tool.execution.completed", { name: "call_api", executionId: id, status }),
  ];
  const source = () => call("source-read", { operationId: "GET /api/issues/{id}", pathParams: { id: sourceIssueId } }, {
    ok: true, status: 200, apiOperationId: "GET /api/issues/{id}", artifact: {
      artifactId: assetId, byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex"),
      url: `/api/assets/${assetId}/content`,
    },
  });
  const page = (offset: number, id = `page-${offset}`, artifactId = assetId) => {
    const end = Math.min(offset + 8192, bytes.length);
    return call(id, { operationId: operation, pathParams: { assetId: artifactId }, responseText: { offsetBytes: offset, limitBytes: 8192 } }, {
      ok: true, status: 200, apiOperationId: operation, data: bytes.subarray(offset, end).toString(),
      responseText: { offsetBytes: offset, totalBytes: bytes.length, nextOffsetBytes: end === bytes.length ? null : end },
    });
  };
  const pages = () => Array.from({ length: Math.ceil(bytes.length / 8192) }, (_, i) => page(i * 8192)).flat();
  it("accepts correlated source artifact pages through EOF, including a harmless identical retry", () => {
    expect(gradeApiResponsePaging([...source(), ...page(0, "retry"), ...pages()], sourceIssueId).passed).toBe(true);
  });
  it("rejects unrelated completed calls", () => {
    const unrelated = [1, 2, 3].flatMap(i => call(`unrelated-${i}`, { operationId: "GET /api/agents/me" }, { ok: true, status: 200 }));
    expect(gradeApiResponsePaging(unrelated, sourceIssueId).passed).toBe(false);
  });
  it("rejects repeating the first page without reaching EOF", () => {
    expect(gradeApiResponsePaging([...source(), ...page(0), ...page(0, "again"), ...page(0, "again-2")], sourceIssueId).passed).toBe(false);
  });
  it("rejects a gap even when the last page claims EOF", () => {
    expect(gradeApiResponsePaging([...source(), ...pages().filter((_e, i) => i < 3 || i >= 6)], sourceIssueId).passed).toBe(false);
  });
  it("rejects reading a different artifact", () => {
    const other = Array.from({ length: Math.ceil(bytes.length / 8192) }, (_, i) => page(i * 8192, `other-${i}`, "other-asset")).flat();
    expect(gradeApiResponsePaging([...source(), ...other], sourceIssueId).passed).toBe(false);
  });
  it("requires the exact seeded source issue", () => {
    expect(gradeApiResponsePaging([...source(), ...pages()], "other-issue").passed).toBe(false);
  });
  it("requires a matched successful completion for every page", () => {
    const incomplete = pages().filter(e => e.eventType !== "tool.execution.completed");
    expect(gradeApiResponsePaging([...source(), ...incomplete], sourceIssueId).passed).toBe(false);
  });
  it.each([
    ["requested offset", { offsetBytes: 1, totalBytes: bytes.length, nextOffsetBytes: 8192 }],
    ["next offset", { offsetBytes: 0, totalBytes: bytes.length, nextOffsetBytes: 8193 }],
    ["total size", { offsetBytes: 0, totalBytes: bytes.length + 1, nextOffsetBytes: 8192 }],
    ["premature EOF", { offsetBytes: 0, totalBytes: bytes.length, nextOffsetBytes: null }],
  ])("rejects mismatched %s metadata", (_label, responseText) => {
    const malformed = call("bad-page", {
      operationId: operation, pathParams: { assetId }, responseText: { offsetBytes: 0, limitBytes: 8192 },
    }, { ok: true, status: 206, apiOperationId: operation, data: bytes.subarray(0, 8192).toString(), responseText });
    expect(gradeApiResponsePaging([...source(), ...pages(), ...malformed], sourceIssueId).passed).toBe(false);
  });
  it("rejects contradictory duplicate bytes and a wrong source artifact digest", () => {
    const corrupt = call("corrupt-page", {
      operationId: operation, pathParams: { assetId }, responseText: { offsetBytes: 0, limitBytes: 8192 },
    }, {
      ok: true, status: 206, apiOperationId: operation, data: "x".repeat(8192),
      responseText: { offsetBytes: 0, totalBytes: bytes.length, nextOffsetBytes: 8192 },
    });
    expect(gradeApiResponsePaging([...source(), ...pages(), ...corrupt], sourceIssueId).passed).toBe(false);
    expect(gradeApiResponsePaging([...source(), ...corrupt, ...pages().slice(3)], sourceIssueId).passed).toBe(false);
  });
});