fix(runner): read saved API responses in bounded text pages

Keep large response artifacts inspectable through the existing authenticated API route. Preserve company and run authority, UTF-8 boundaries, and transfer limits. Reproduce the artifact loop through real HTTP and add an explicit full-stack fixture.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/services/native-runtime/runner-api-client.ts

@@ -18,6 +18,10 @@ export const runnerApiCallSchema = z.object({
   body: z.unknown().optional(),
   contentType: z.string().max(120).optional(),
   files: z.array(fileSchema).max(10).optional(),
+  responseText: z.object({
+    offsetBytes: z.number().int().min(0).max(RUNNER_API_MAX_BYTES).optional(),
+    limitBytes: z.number().int().min(4).max(RUNNER_API_INLINE_BYTES).optional(),
+  }).strict().optional(),
 }).strict();
 export type RunnerApiCall = z.infer<typeof runnerApiCallSchema>;
 export type RunnerApiFile = z.infer<typeof fileSchema>;
@@ -44,6 +48,7 @@ export function validateRunnerApiCall(value: unknown, context: RunnerApiContext)
   const input = parsed.data;
   const operation = runnerApiOperation(input.operationId);
   if (operation.transport !== "rest") throw unprocessable("This endpoint requires its existing protocol client; call_api supports REST only");
+  if (input.responseText && operation.method !== "GET") throw badRequest("responseText requires a GET operation; read the saved artifact instead of repeating a mutation");
   const restriction = runnerApiRestriction(operation.method, operation.path);
   if (restriction) throw forbidden(restriction);
   if (!operation.allowedModes.includes(context.workMode)) throw forbidden("call_api permits only reads in Ask and Plan modes; use the permitted dedicated tools");
@@ -139,7 +144,8 @@ export async function readBoundedResponse(response: Response, maxBytes = RUNNER_
 }
 
 export async function executeRunnerApi(input: RunnerApiCall, context: RunnerApiContext, io: RunnerApiIo) {
-  const { operation } = validateRunnerApiCall(input, context);
+  const { operation, input: validated } = validateRunnerApiCall(input, context);
+  input = validated;
   const url = runnerApiUrl(operation, input, context, io.apiUrl);
   if (!io.token) throw new Error("Paperclip run authentication is unavailable");
   const headers = new Headers({ Authorization: `Bearer ${io.token}`, "X-Paperclip-Run-Id": context.runId });
@@ -192,9 +198,27 @@ export async function executeRunnerApi(input: RunnerApiCall, context: RunnerApiC
   const uncertainty = { outcome: "unknown", guidance: "Inspect current state before retrying this mutation; it may already have succeeded." };
   const base = { ok: response.ok, status: response.status, operationId: operation.operationId, contentType: type, retryAfter: response.headers.get("retry-after"), ...(uncertain ? uncertainty : {}) };
   if (response.status >= 300 && response.status < 400) return { ...base, ok: false, error: "api_redirect_not_followed" };
+  if (input.responseText) {
+    if (!/json|^text\//i.test(type)) return { ...base, ok: false, error: "response_text_requires_text_content" };
+    const offsetBytes = input.responseText.offsetBytes ?? 0;
+    if (offsetBytes > bytes.length || (offsetBytes < bytes.length && (bytes[offsetBytes] & 0xc0) === 0x80)) {
+      return { ...base, ok: false, error: "response_text_invalid_offset", byteSize: bytes.length };
+    }
+    let end = Math.min(bytes.length, offsetBytes + (input.responseText.limitBytes ?? RUNNER_API_INLINE_BYTES));
+    // Do not cut a UTF-8 code point. The next offset always starts a whole one.
+    while (end < bytes.length && end > offsetBytes && (bytes[end] & 0xc0) === 0x80) end--;
+    if (end === offsetBytes && end < bytes.length) return { ...base, ok: false, error: "response_text_invalid_utf8" };
+    try {
+      const data = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes.subarray(offsetBytes, end));
+      return { ...base, data, responseText: { offsetBytes, nextOffsetBytes: end < bytes.length ? end : null, totalBytes: bytes.length } };
+    } catch {
+      return { ...base, ok: false, error: "response_text_invalid_utf8" };
+    }
+  }
   if (!bytes.length) return { ...base, data: null };
   if (bytes.length > RUNNER_API_INLINE_BYTES || !/json|^text\//i.test(type)) {
-    return { ...base, artifact: await io.saveResponse(bytes, type), byteSize: bytes.length, preview: /json|^text\//i.test(type) ? bytes.subarray(0, 2000).toString("utf8") : null };
+    return { ...base, artifact: await io.saveResponse(bytes, type), byteSize: bytes.length, preview: /json|^text\//i.test(type) ? bytes.subarray(0, 2000).toString("utf8") : null,
+      ...(/json|^text\//i.test(type) ? { guidance: "Read the saved artifact with GET /api/assets/{assetId}/content and responseText: { offsetBytes: 0 }. Continue with responseText.nextOffsetBytes until null. Do not repeat a mutation to read its response." } : {}) };
   }
   const text = bytes.toString("utf8");
   if (/json/i.test(type)) {

server/src/services/native-runtime/runner-api.integration.test.ts

@@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto";
 import { chmod, writeFile, symlink } from "node:fs/promises";
 import { join } from "node:path";
 import { eq } from "drizzle-orm";
-import { documents, heartbeatRuns, issues, routineDocuments, routines } from "@paperclipai/db";
+import { assets, documents, heartbeatRuns, issues, routineDocuments, routines } from "@paperclipai/db";
 import { beforeAll, afterAll, beforeEach, afterEach, describe, expect, it, vi } from "vitest";
 import { startRunnerApiTestServer } from "../../__tests__/helpers/runner-api-server.js";
 import { createRunnerdCodexTransport, defaultCapabilityRunnerdBinary } from "../../vendor/paperclip-runner/index.js";
@@ -248,6 +248,29 @@ else if(m.id!==undefined) send({id:m.id,result:{}});
     } finally { getObject.mockRestore(); }
   });
 
+  it("retrieves every byte of a saved large response without creating an artifact loop", async () => {
+    const fixture = await server.fixture();
+    const foreign = await server.fixture();
+    const text = JSON.stringify({ padding: "🧭é".repeat(6000), evidence: "last-field-is-readable" });
+    const stored = await server.storage.putFile({ companyId: fixture.companyId, namespace: "eval", originalFilename: "evidence.json", contentType: "application/json", body: Buffer.from(text) });
+    const [asset] = await server.db.insert(assets).values({ ...stored, companyId: fixture.companyId, createdByAgentId: fixture.agentId }).returning();
+    const call = (authority: typeof fixture.authority, assetId: string, responseText?: object) => authority.execute({ tool: "call_api", callId: randomUUID(), arguments: { operationId: "GET /api/assets/{assetId}/content", pathParams: { assetId }, ...(responseText ? { responseText } : {}) } }) as Promise<any>;
+    const initial = await call(fixture.authority, asset.id);
+    expect(initial).toMatchObject({ status: 200, artifact: { byteSize: Buffer.byteLength(text) } });
+    const before = (await fixture.snapshot()).assets.length;
+    let offsetBytes = 0;
+    let result = "";
+    do {
+      const page = await call(fixture.authority, initial.artifact.artifactId, { offsetBytes, limitBytes: 4096 });
+      expect(page).toMatchObject({ status: 200, responseText: { offsetBytes, totalBytes: Buffer.byteLength(text) } });
+      result += page.data;
+      offsetBytes = page.responseText.nextOffsetBytes;
+    } while (offsetBytes !== null);
+    expect(JSON.parse(result)).toEqual(JSON.parse(text));
+    expect((await fixture.snapshot()).assets).toHaveLength(before);
+    expect(await call(foreign.authority, initial.artifact.artifactId, { offsetBytes: 0 })).toMatchObject({ ok: false, status: 404 });
+  });
+
   it("contains workspace files, checks artifact ownership, and persists downloads", async () => {
     const fixture = await server.fixture();
     const foreign = await server.fixture();

server/src/services/native-runtime/runner-api.test.ts

@@ -591,6 +591,51 @@ describe("runner API request boundary", () => {
       ).toThrow("inline JSON object");
     }
   });
+  it("reads complete large text without recursively saving response artifacts", async () => {
+    const text = "prefix\n" + "🧭é\n".repeat(6000) + "END-OF-EVIDENCE";
+    const saveResponse = vi.fn(io(fetch).saveResponse);
+    const request = vi.fn<typeof fetch>(async () => new Response(text, { headers: { "content-type": "application/json" } }));
+    let offsetBytes = 0;
+    let received = "";
+    do {
+      const result = await executeRunnerApi({ operationId: projects, responseText: { offsetBytes, limitBytes: 4096 } }, context, { ...io(request), saveResponse }) as any;
+      expect(result).toMatchObject({ ok: true, responseText: { offsetBytes, totalBytes: Buffer.byteLength(text) } });
+      expect(Buffer.byteLength(result.data)).toBeLessThanOrEqual(4096);
+      received += result.data;
+      offsetBytes = result.responseText.nextOffsetBytes;
+    } while (offsetBytes !== null);
+    expect(received).toBe(text);
+    expect(saveResponse).not.toHaveBeenCalled();
+    expect(request.mock.calls.every(([, options]) => new Headers(options?.headers).get("Authorization") === "Bearer private-agent-token")).toBe(true);
+  });
+  it.each([{ offsetBytes: -1 }, { limitBytes: 0 }, { limitBytes: 3 }, { limitBytes: 24577 }, { offsetBytes: 0.5 }])("rejects invalid text windows before dispatch: %j", async responseText => {
+    const request = vi.fn<typeof fetch>();
+    await expect(executeRunnerApi({ operationId: projects, responseText }, context, io(request))).rejects.toThrow("Invalid call_api arguments");
+    expect(request).not.toHaveBeenCalled();
+  });
+  it("does not page mutation responses by repeating the mutation", async () => {
+    const request = vi.fn<typeof fetch>();
+    await expect(executeRunnerApi({ operationId: createProject, body: { name: "Once" }, responseText: {} }, context, io(request))).rejects.toThrow("GET");
+    expect(request).not.toHaveBeenCalled();
+  });
+  it("rejects binary, invalid UTF-8 and offsets that split text", async () => {
+    for (const [body, type, offsetBytes] of [
+      [Buffer.from("binary"), "application/octet-stream", 0],
+      [Buffer.from([0xff]), "text/plain", 0],
+      [Buffer.concat([Buffer.from([0xc2]), Buffer.alloc(24576, 0x80)]), "text/plain", 0],
+      [Buffer.from("é"), "text/plain", 1],
+      [Buffer.from("short"), "text/plain", 6],
+    ] as const) {
+      const saveResponse = vi.fn(io(fetch).saveResponse);
+      const result = await executeRunnerApi({ operationId: projects, responseText: { offsetBytes } }, context, { ...io(async () => new Response(body, { headers: { "content-type": type } })), saveResponse });
+      expect(result).toMatchObject({ ok: false, error: expect.stringContaining("response_text") });
+      expect(saveResponse).not.toHaveBeenCalled();
+    }
+  });
+  it("preserves denials and the transfer limit for bounded text reads", async () => {
+    expect(await executeRunnerApi({ operationId: projects, responseText: {} }, context, io(async () => Response.json({ error: "denied" }, { status: 403 })))).toMatchObject({ ok: false, status: 403, data: '{"error":"denied"}' });
+    expect(await executeRunnerApi({ operationId: projects, responseText: {} }, context, io(async () => new Response("small", { headers: { "content-type": "text/plain", "content-length": String(10 * 1024 * 1024 + 1) } })))).toMatchObject({ ok: false, status: null, outcome: "read_failed" });
+  });
   it("bounds streamed responses even without content-length", async () => {
     await expect(
       readBoundedResponse(new Response("too large"), 3),