Bound saved API text pages with authenticated storage ranges

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/runner-api-tools.md

@@ -69,6 +69,12 @@ and offsets inside a code point or beyond the response. This option only works
 with GET; never repeat a mutation to retrieve another part of its response.
 Read the saved artifact for a stable snapshot instead of paging a changing live
 response. The existing 10 MiB transfer limit and all route authorization remain.
+Saved asset pages use authenticated HTTP byte ranges. The storage provider reads
+only the requested window, with at most two extra bytes for UTF-8/EOF handling.
+The client validates `Content-Range`, the total size, and the received byte count;
+it rejects unsupported or inconsistent ranges instead of downloading the whole
+asset for every page. Other GET routes retain their existing bounded full-response
+read, so use the saved asset for large responses.
 
 Tool responses identify the HTTP route with `apiOperationId`. The native protocol
 reserves `operationId` and `callId` for semantic tool-call identity; API metadata

server/src/__tests__/assets.test.ts

@@ -421,6 +421,29 @@ describe("GET /api/assets/:assetId/content", () => {
     getAssetByIdMock.mockReset();
   });
 
+  it("reads only the requested storage range for a large asset", async () => {
+    const storage = createStorageService("text/plain");
+    getAssetByIdMock.mockResolvedValue({ ...createAsset(), contentType: "text/plain", byteSize: 10 * 1024 * 1024 });
+    vi.mocked(storage.getObject).mockImplementation(async (_company, _key, options) => {
+      expect(options).toEqual({ range: { start: 8192, end: 12288 } });
+      return { stream: Readable.from(Buffer.alloc(4097, 65)), contentLength: 4097 };
+    });
+    const res = await requestApp(await createApp(storage), baseUrl => request(baseUrl).get("/api/assets/asset-1/content").set("Range", "bytes=8192-12288"));
+    expect(res.status).toBe(206);
+    expect(res.headers["content-range"]).toBe("bytes 8192-12288/10485760");
+    expect(res.headers["content-length"]).toBe("4097");
+    expect(res.text).toBe("A".repeat(4097));
+  });
+
+  it.each(["bytes=999-1000", "bytes=0-1,3-4", "bytes=1x-3", "bytes=-", "items=0-3"])("rejects invalid asset ranges before storage reads: %s", async range => {
+    const storage = createStorageService();
+    getAssetByIdMock.mockResolvedValue(createAsset());
+    const res = await requestApp(await createApp(storage), baseUrl => request(baseUrl).get("/api/assets/asset-1/content").set("Range", range));
+    expect(res.status).toBe(416);
+    expect(res.headers["content-range"]).toBe("bytes */40");
+    expect(storage.getObject).not.toHaveBeenCalled();
+  });
+
   it("downloads script-capable HTML with nosniff and a sandbox CSP", async () => {
     const html = Buffer.from("<script>globalThis.__assetXss = true</script>");
     const storage = createStorageService("text/html");

server/src/routes/assets.ts

@@ -328,13 +328,31 @@ export function assetRoutes(db: Db, storage: StorageService) {
     const asset = await getAccessibleResource(req, res, svc.getById(assetId), "Asset not found");
     if (!asset) return;
 
-    const object = await storage.getObject(asset.companyId, asset.objectKey);
+    // Use the persisted size only after resource authorization. Single ranges
+    // keep saved API text pages bounded all the way to disk or object storage.
+    const rawRange = req.headers.range;
+    const rangeSyntax = rawRange && /^bytes=(\d*)-(\d*)$/i.exec(rawRange);
+    const emptyRead = asset.byteSize === 0 && rangeSyntax?.[1] === "0";
+    const ranges = rawRange && !emptyRead ? req.range(asset.byteSize) : undefined;
+    res.setHeader("Accept-Ranges", "bytes");
+    if (rawRange && (!rangeSyntax || (!rangeSyntax[1] && !rangeSyntax[2])
+      || (!emptyRead && (!Array.isArray(ranges) || ranges.length !== 1)))) {
+      res.setHeader("Content-Range", `bytes */${asset.byteSize}`);
+      res.status(416).end();
+      return;
+    }
+    const range = Array.isArray(ranges) ? ranges[0] : undefined;
+    const object = await storage.getObject(asset.companyId, asset.objectKey, range ? { range } : undefined);
     const responseContentType = asset.contentType || object.contentType || "application/octet-stream";
     const mediaType = responseContentType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
     const inlineSafe = mediaType !== SVG_CONTENT_TYPE
       && isInlineAttachmentContentType(mediaType);
     res.setHeader("Content-Type", responseContentType);
-    res.setHeader("Content-Length", String(asset.byteSize || object.contentLength || 0));
+    res.setHeader("Content-Length", String(range ? range.end - range.start + 1 : asset.byteSize || object.contentLength || 0));
+    if (range) {
+      res.status(206);
+      res.setHeader("Content-Range", `bytes ${range.start}-${range.end}/${asset.byteSize}`);
+    }
     res.setHeader("Cache-Control", "private, max-age=60");
     res.setHeader("X-Content-Type-Options", "nosniff");
     if (!inlineSafe) {
@@ -349,6 +367,7 @@ export function assetRoutes(db: Db, storage: StorageService) {
     object.stream.on("error", (err) => {
       next(err);
     });
+    res.on("close", () => object.stream.destroy());
     object.stream.pipe(res);
   });
 

server/src/services/native-runtime/runner-api-client.ts

@@ -149,6 +149,14 @@ export async function executeRunnerApi(input: RunnerApiCall, context: RunnerApiC
   const url = runnerApiUrl(operation, input, context, io.apiUrl);
   if (!io.token) throw new Error("Paperclip run authentication is unavailable");
   const headers = new Headers({ Authorization: `Bearer ${io.token}`, "X-Paperclip-Run-Id": context.runId });
+  const textOffset = input.responseText?.offsetBytes ?? 0;
+  const textLimit = input.responseText?.limitBytes ?? RUNNER_API_INLINE_BYTES;
+  const assetTextRange = input.responseText && operation.operationId === "GET /api/assets/{assetId}/content"
+    ? { start: Math.max(0, textOffset - 1), end: textOffset + textLimit }
+    : undefined;
+  // One byte before the requested offset permits an EOF read; one byte after
+  // the page lets the decoder trim an incomplete UTF-8 code point.
+  if (assetTextRange) headers.set("Range", `bytes=${assetTextRange.start}-${assetTextRange.end}`);
   let body: BodyInit | undefined;
   const contentType = input.contentType ?? (input.files?.length ? "multipart/form-data" : "application/json");
   if (/\r|\n/.test(contentType)) throw badRequest("Invalid content type");
@@ -186,7 +194,7 @@ export async function executeRunnerApi(input: RunnerApiCall, context: RunnerApiC
   await io.beforeDispatch?.();
   try {
     response = await (io.fetch ?? fetch)(url, { method: operation.method, headers, body, redirect: "manual", signal: AbortSignal.timeout(RUNNER_API_TIMEOUT_MS) });
-    bytes = await readBoundedResponse(response);
+    bytes = await readBoundedResponse(response, assetTextRange && response.ok ? textLimit + 2 : RUNNER_API_MAX_BYTES);
   } catch {
     return { ok: false, status: null, operationId: operation.operationId, error: "api_transport_failure", outcome: ["GET", "HEAD", "OPTIONS"].includes(operation.method) ? "read_failed" : "unknown", guidance: "Inspect current state before retrying a mutation; it may already have succeeded." };
   }
@@ -200,17 +208,39 @@ export async function executeRunnerApi(input: RunnerApiCall, context: RunnerApiC
   if (response.status >= 300 && response.status < 400) return { ...base, ok: false, error: "api_redirect_not_followed" };
   if (input.responseText) {
     if (!/json|^text\//i.test(type)) return { ...base, ok: false, error: "response_text_requires_text_content" };
-    const offsetBytes = input.responseText.offsetBytes ?? 0;
-    if (offsetBytes > bytes.length || (offsetBytes < bytes.length && (bytes[offsetBytes] & 0xc0) === 0x80)) {
-      return { ...base, ok: false, error: "response_text_invalid_offset", byteSize: bytes.length };
+    let totalBytes = bytes.length;
+    let baseOffset = 0;
+    if (assetTextRange && response.ok) {
+      const range = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(response.headers.get("content-range") ?? "");
+      if (response.status === 200 && bytes.length === 0 && textOffset === 0) {
+        // An empty asset has no satisfiable byte range; its route returns 200.
+      } else if (response.status !== 206 || !range) {
+        return { ...base, ok: false, error: "response_text_invalid_range" };
+      } else {
+        const [start, end, total] = range.slice(1).map(Number);
+        if (![start, end, total].every(Number.isSafeInteger) || start !== assetTextRange.start
+          || total > RUNNER_API_MAX_BYTES || total < 1 || end !== Math.min(assetTextRange.end, total - 1)
+          || end < start || bytes.length !== end - start + 1) {
+          return { ...base, ok: false, error: "response_text_invalid_range" };
+        }
+        baseOffset = start;
+        totalBytes = total;
+      }
     }
-    let end = Math.min(bytes.length, offsetBytes + (input.responseText.limitBytes ?? RUNNER_API_INLINE_BYTES));
+    // HTTP denials are evidence, not text pages of an authorized asset.
+    const offsetBytes = response.ok ? textOffset : 0;
+    const localOffset = offsetBytes - baseOffset;
+    if (offsetBytes > totalBytes || localOffset < 0 || (localOffset < bytes.length && (bytes[localOffset] & 0xc0) === 0x80)) {
+      return { ...base, ok: false, error: "response_text_invalid_offset", byteSize: totalBytes };
+    }
+    let end = Math.min(bytes.length, localOffset + textLimit);
     // Do not cut a UTF-8 code point. The next offset always starts a whole one.
-    while (end < bytes.length && end > offsetBytes && (bytes[end] & 0xc0) === 0x80) end--;
-    if (end === offsetBytes && end < bytes.length) return { ...base, ok: false, error: "response_text_invalid_utf8" };
+    while (end < bytes.length && end > localOffset && (bytes[end] & 0xc0) === 0x80) end--;
+    if (end === localOffset && end < bytes.length) return { ...base, ok: false, error: "response_text_invalid_utf8" };
     try {
-      const data = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes.subarray(offsetBytes, end));
-      return { ...base, data, responseText: { offsetBytes, nextOffsetBytes: end < bytes.length ? end : null, totalBytes: bytes.length } };
+      const data = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes.subarray(localOffset, end));
+      const nextOffset = baseOffset + end;
+      return { ...base, data, responseText: { offsetBytes, nextOffsetBytes: nextOffset < totalBytes ? nextOffset : null, totalBytes } };
     } catch {
       return { ...base, ok: false, error: "response_text_invalid_utf8" };
     }

server/src/services/native-runtime/runner-api.integration.test.ts

@@ -258,17 +258,26 @@ else if(m.id!==undefined) send({id:m.id,result:{}});
     const initial = await call(fixture.authority, asset.id);
     expect(initial).toMatchObject({ status: 200, artifact: { byteSize: Buffer.byteLength(text) } });
     const before = (await fixture.snapshot()).assets.length;
+    const reads = vi.spyOn(server.storage, "getObject");
     let offsetBytes = 0;
     let result = "";
-    do {
-      const page = await call(fixture.authority, initial.artifact.artifactId, { offsetBytes, limitBytes: 4096 });
-      expect(page).toMatchObject({ status: 200, responseText: { offsetBytes, totalBytes: Buffer.byteLength(text) } });
-      result += page.data;
-      offsetBytes = page.responseText.nextOffsetBytes;
-    } while (offsetBytes !== null);
-    expect(JSON.parse(result)).toEqual(JSON.parse(text));
-    expect((await fixture.snapshot()).assets).toHaveLength(before);
-    expect(await call(foreign.authority, initial.artifact.artifactId, { offsetBytes: 0 })).toMatchObject({ ok: false, status: 404 });
+    try {
+      do {
+        const page = await call(fixture.authority, initial.artifact.artifactId, { offsetBytes, limitBytes: 4096 });
+        expect(page).toMatchObject({ status: 206, responseText: { offsetBytes, totalBytes: Buffer.byteLength(text) } });
+        result += page.data;
+        offsetBytes = page.responseText.nextOffsetBytes;
+      } while (offsetBytes !== null);
+      expect(JSON.parse(result)).toEqual(JSON.parse(text));
+      expect(reads.mock.calls.length).toBeGreaterThan(1);
+      expect(reads.mock.calls.every(([, , options]) => options?.range)).toBe(true);
+      const storageBytes = reads.mock.calls.reduce((total, [, , options]) => total + options!.range!.end - options!.range!.start + 1, 0);
+      expect(storageBytes).toBeLessThan(Buffer.byteLength(text) + reads.mock.calls.length * 5);
+      expect((await fixture.snapshot()).assets).toHaveLength(before);
+      expect(await call(foreign.authority, initial.artifact.artifactId, { offsetBytes: 0 })).toMatchObject({ ok: false, status: 404 });
+    } finally {
+      reads.mockRestore();
+    }
   });
 
   it("contains workspace files, checks artifact ownership, and persists downloads", async () => {

server/src/services/native-runtime/runner-api.test.ts

@@ -608,6 +608,48 @@ describe("runner API request boundary", () => {
     expect(saveResponse).not.toHaveBeenCalled();
     expect(request.mock.calls.every(([, options]) => new Headers(options?.headers).get("Authorization") === "Bearer private-agent-token")).toBe(true);
   });
+  it("pages a ten MiB asset with linear transfer and UTF-8 boundaries", async () => {
+    const bytes = Buffer.from("🧭é".repeat(Math.floor(10 * 1024 * 1024 / 6)));
+    let transferred = 0;
+    const request = vi.fn<typeof fetch>(async (_url, init) => {
+      const range = new Headers(init?.headers).get("range");
+      const match = /^bytes=(\d+)-(\d+)$/.exec(range ?? "");
+      if (!match) { transferred += bytes.length; return new Response(bytes, { headers: { "content-type": "text/plain" } }); }
+      const start = Number(match[1]), end = Math.min(Number(match[2]), bytes.length - 1);
+      const part = bytes.subarray(start, end + 1); transferred += part.length;
+      return new Response(part, { status: 206, headers: { "content-type": "text/plain", "content-range": `bytes ${start}-${end}/${bytes.length}` } });
+    });
+    let offsetBytes: number | null = 0;
+    const parts: string[] = [];
+    do {
+      const page: any = await executeRunnerApi({ operationId: "GET /api/assets/{assetId}/content", pathParams: { assetId: "large" }, responseText: { offsetBytes } }, context, io(request));
+      expect(page.ok).toBe(true);
+      parts.push(page.data); offsetBytes = page.responseText.nextOffsetBytes;
+    } while (offsetBytes !== null);
+    expect(parts.join("")).toBe(bytes.toString());
+    expect(transferred).toBeLessThan(bytes.length + request.mock.calls.length * 5);
+    expect(request.mock.calls.every(([, init]) => new Headers(init?.headers).has("range"))).toBe(true);
+  });
+  it.each([
+    ["bytes 4-8/20", "12345"], // wrong start
+    ["bytes 3-7/20", "12345"], // early end
+    ["bytes 3-8/20", "12345"], // truncated body
+    ["bytes 3-8/10485761", "123456"], // total exceeds transfer policy
+    ["bytes 3-8/*", "123456"], // unknown total
+    [null, "123456"],
+  ])("rejects inconsistent partial asset receipts: %s", async (range, body) => {
+    const result = await executeRunnerApi({ operationId: "GET /api/assets/{assetId}/content", pathParams: { assetId: "saved" }, responseText: { offsetBytes: 4, limitBytes: 4 } }, context, io(async () => new Response(body, { status: 206, headers: { "content-type": "text/plain", ...(range ? { "content-range": range } : {}) } })));
+    expect(result).toMatchObject({ ok: false, error: "response_text_invalid_range" });
+  });
+  it("accepts EOF and rejects an offset inside a UTF-8 code point in ranged assets", async () => {
+    const call = (offsetBytes: number, bytes: Buffer, range: string) => executeRunnerApi({ operationId: "GET /api/assets/{assetId}/content", pathParams: { assetId: "saved" }, responseText: { offsetBytes, limitBytes: 4 } }, context, io(async () => new Response(new Uint8Array(bytes), { status: 206, headers: { "content-type": "text/plain", "content-range": range } })));
+    expect(await call(2, Buffer.from([0xa9]), "bytes 1-1/2")).toMatchObject({ ok: true, data: "", responseText: { nextOffsetBytes: null, totalBytes: 2 } });
+    expect(await call(1, Buffer.from("é"), "bytes 0-1/2")).toMatchObject({ ok: false, error: "response_text_invalid_offset" });
+  });
+  it("does not silently accept an asset server that ignores the byte range", async () => {
+    const result = await executeRunnerApi({ operationId: "GET /api/assets/{assetId}/content", pathParams: { assetId: "saved" }, responseText: { limitBytes: 4 } }, context, io(async () => new Response("unbounded", { headers: { "content-type": "text/plain" } })));
+    expect(result).toMatchObject({ ok: false, error: "api_transport_failure" });
+  });
   it.each([{ offsetBytes: -1 }, { limitBytes: 0 }, { limitBytes: 3 }, { limitBytes: 24577 }, { offsetBytes: 0.5 }])("rejects invalid text windows before dispatch: %j", async responseText => {
     const request = vi.fn<typeof fetch>();
     await expect(executeRunnerApi({ operationId: projects, responseText }, context, io(request))).rejects.toThrow("Invalid call_api arguments");