test: verify downloaded API response proof bytes

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/api-response-reading.test.ts

@@ -1,5 +1,7 @@
-import { describe, expect, it } from "vitest";
-import { apiResponseReadingTask, responseEvidenceCode, responseEvidenceDescription, successfulApiReadCount } from "./api-response-reading.js";
+import { createHash } from "node:crypto";
+import { describe, expect, it, vi } from "vitest";
+import type { RunnerApi } from "./api.js";
+import { apiResponseReadingTask, readResponseProof, responseEvidenceCode, responseEvidenceDescription, successfulApiReadCount } from "./api-response-reading.js";
 
 describe("large API response evidence fixture", () => {
   it("keeps evidence beyond both preview and inline limits and out of the assignment", () => {
@@ -16,4 +18,19 @@ describe("large API response evidence fixture", () => {
     expect(successfulApiReadCount([])).toBe(0);
     expect(successfulApiReadCount([{ ...event, eventType: "item.delta" }, { ...event, payload: null }, { ...event, payload: { prpEvent: { payload: { name: "call_api", status: "failed" } } } }])).toBe(0);
   });
+  it("downloads exact run-attributed proof bytes rather than trusting the agent's report", async () => {
+    const bytes = Buffer.from("independent evidence\n");
+    const proof = { id: "proof", issueId: "issue", originatingRunId: "run", originalFilename: "api-response-proof.txt", byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") };
+    const download = vi.fn().mockResolvedValue({ ok: () => true, body: async () => bytes });
+    const get = vi.fn().mockResolvedValue([proof]);
+    const api = { get, request: { get: download } } as unknown as RunnerApi;
+    expect((await readResponseProof(api, "issue", "run")).content).toBe(bytes.toString());
+    expect(download).toHaveBeenCalledWith("/api/attachments/proof/content?download=1");
+    get.mockResolvedValue([{ ...proof, originatingRunId: "other" }]);
+    await expect(readResponseProof(api, "issue", "run")).rejects.toThrow("observed 0");
+    get.mockResolvedValue([proof, proof]);
+    await expect(readResponseProof(api, "issue", "run")).rejects.toThrow("observed 2");
+    get.mockResolvedValue([{ ...proof, sha256: "mismatch" }]);
+    await expect(readResponseProof(api, "issue", "run")).rejects.toThrow("disagrees");
+  });
 });

tests/runner-e2e/api-response-reading.ts

@@ -1,5 +1,24 @@
 import { createHash } from "node:crypto";
 import type { RunnerTaskFixture } from "./types.js";
+import type { RunnerApi } from "./api.js";
+import type { IssueAttachment } from "../../packages/shared/src/types/issue.js";
+
+/** Verify delivered bytes through the same public download path the user receives. */
+export async function readResponseProof(api: RunnerApi, issueId: string, runId: string) {
+  const attachments = await api.get<IssueAttachment[]>(`/api/issues/${issueId}/attachments`);
+  const proofs = attachments.filter(attachment =>
+    attachment.issueId === issueId && attachment.originatingRunId === runId &&
+    attachment.originalFilename === "api-response-proof.txt");
+  if (proofs.length !== 1) throw new Error(`Expected one proof attachment from the tested run; observed ${proofs.length}`);
+  const proof = proofs[0]!;
+  const response = await api.request.get(`/api/attachments/${encodeURIComponent(proof.id)}/content?download=1`);
+  if (!response.ok()) throw new Error(`Proof download returned ${response.status()}`);
+  const bytes = await response.body();
+  if (bytes.length !== proof.byteSize || createHash("sha256").update(bytes).digest("hex") !== proof.sha256) {
+    throw new Error("Downloaded proof disagrees with stored attachment bytes");
+  }
+  return { attachmentId: proof.id, content: bytes.toString("utf8"), sha256: proof.sha256 };
+}
 
 export function responseEvidenceCode(nonce: string) {
   return createHash("sha256").update(`bounded-response-evidence:${nonce}`).digest("hex");
@@ -29,7 +48,7 @@ export const apiResponseReadingTask: RunnerTaskFixture = {
     "Read the saved artifact through GET /api/assets/{assetId}/content with responseText: {offsetBytes:0,limitBytes:8192}.",
     "Continue using responseText.nextOffsetBytes until null. Extract the Evidence code at the end of its description.",
     "The large response must be read with bounded responseText pages. Do not use other tools or API routes to obtain the evidence; if bounded reading fails, report the failure instead of substituting a different reader.",
-    "Write only that code followed by a newline to api-response-proof.txt in the current workspace. Do not edit the evidence task or create child tasks.",
+    "Write only that code followed by a newline to api-response-proof.txt in the current workspace and deliver that file as an attachment. Do not edit the evidence task or create child tasks.",
     `Verify the proof file and finish this task with paperclip_finish, reportedWorkDisposition done, and summary API_RESPONSE_READ_${nonce}.`,
   ].join("\n"),
   buildMatchers: (nonce, execution) => [

tests/runner-e2e/runner.spec.ts

@@ -1,4 +1,4 @@
-import { responseEvidenceDescription, successfulApiReadCount } from "./api-response-reading.js";
+import { readResponseProof, responseEvidenceDescription, successfulApiReadCount } from "./api-response-reading.js";
 import { observeBrowserBootstrap } from "./browser-bootstrap-diagnostics.js";
 import { runAccountingFlow } from "./accounting-flow.js";
 import type { Issue } from "../../packages/shared/src/types/issue.js";
@@ -1952,6 +1952,11 @@ for (const execution of executions) {
             ]),
         ),
       );
+      if (execution.suite.id === "api-response-reading") {
+        const proof = await readResponseProof(api, issue.id, run.id);
+        fileObservations["api-response-proof.txt"] = proof.content;
+        await writeSanitizedJson(snapshotsDir, "downloaded-response-proof.json", proof, secrets);
+      }
       matcherResults = await Promise.all(
         taskMatchers.map((matcher) =>
           evaluateMatcher(matcher, {