diff --git a/doc/runner-api-tools.md b/doc/runner-api-tools.md index 2f72b0662be..8e533cdd4df 100644 --- a/doc/runner-api-tools.md +++ b/doc/runner-api-tools.md @@ -69,6 +69,12 @@ and offsets inside a code point or beyond the response. This option only works with GET; never repeat a mutation to retrieve another part of its response. Read the saved artifact for a stable snapshot instead of paging a changing live response. The existing 10 MiB transfer limit and all route authorization remain. +Saved asset pages use authenticated HTTP byte ranges. The storage provider reads +only the requested window, with at most two extra bytes for UTF-8/EOF handling. +The client validates `Content-Range`, the total size, and the received byte count; +it rejects unsupported or inconsistent ranges instead of downloading the whole +asset for every page. Other GET routes retain their existing bounded full-response +read, so use the saved asset for large responses. Tool responses identify the HTTP route with `apiOperationId`. The native protocol reserves `operationId` and `callId` for semantic tool-call identity; API metadata diff --git a/server/src/__tests__/assets.test.ts b/server/src/__tests__/assets.test.ts index 9c8308fb92d..fbc6976498c 100644 --- a/server/src/__tests__/assets.test.ts +++ b/server/src/__tests__/assets.test.ts @@ -421,6 +421,29 @@ describe("GET /api/assets/:assetId/content", () => { getAssetByIdMock.mockReset(); }); + it("reads only the requested storage range for a large asset", async () => { + const storage = createStorageService("text/plain"); + getAssetByIdMock.mockResolvedValue({ ...createAsset(), contentType: "text/plain", byteSize: 10 * 1024 * 1024 }); + vi.mocked(storage.getObject).mockImplementation(async (_company, _key, options) => { + expect(options).toEqual({ range: { start: 8192, end: 12288 } }); + return { stream: Readable.from(Buffer.alloc(4097, 65)), contentLength: 4097 }; + }); + const res = await requestApp(await createApp(storage), baseUrl => request(baseUrl).get("/api/assets/asset-1/content").set("Range", "bytes=8192-12288")); + expect(res.status).toBe(206); + expect(res.headers["content-range"]).toBe("bytes 8192-12288/10485760"); + expect(res.headers["content-length"]).toBe("4097"); + expect(res.text).toBe("A".repeat(4097)); + }); + + it.each(["bytes=999-1000", "bytes=0-1,3-4", "bytes=1x-3", "bytes=-", "items=0-3"])("rejects invalid asset ranges before storage reads: %s", async range => { + const storage = createStorageService(); + getAssetByIdMock.mockResolvedValue(createAsset()); + const res = await requestApp(await createApp(storage), baseUrl => request(baseUrl).get("/api/assets/asset-1/content").set("Range", range)); + expect(res.status).toBe(416); + expect(res.headers["content-range"]).toBe("bytes */40"); + expect(storage.getObject).not.toHaveBeenCalled(); + }); + it("downloads script-capable HTML with nosniff and a sandbox CSP", async () => { const html = Buffer.from(""); const storage = createStorageService("text/html"); diff --git a/server/src/routes/assets.ts b/server/src/routes/assets.ts index 3f9d4ee0c3c..6066b756463 100644 --- a/server/src/routes/assets.ts +++ b/server/src/routes/assets.ts @@ -328,13 +328,31 @@ export function assetRoutes(db: Db, storage: StorageService) { const asset = await getAccessibleResource(req, res, svc.getById(assetId), "Asset not found"); if (!asset) return; - const object = await storage.getObject(asset.companyId, asset.objectKey); + // Use the persisted size only after resource authorization. Single ranges + // keep saved API text pages bounded all the way to disk or object storage. + const rawRange = req.headers.range; + const rangeSyntax = rawRange && /^bytes=(\d*)-(\d*)$/i.exec(rawRange); + const emptyRead = asset.byteSize === 0 && rangeSyntax?.[1] === "0"; + const ranges = rawRange && !emptyRead ? req.range(asset.byteSize) : undefined; + res.setHeader("Accept-Ranges", "bytes"); + if (rawRange && (!rangeSyntax || (!rangeSyntax[1] && !rangeSyntax[2]) + || (!emptyRead && (!Array.isArray(ranges) || ranges.length !== 1)))) { + res.setHeader("Content-Range", `bytes */${asset.byteSize}`); + res.status(416).end(); + return; + } + const range = Array.isArray(ranges) ? ranges[0] : undefined; + const object = await storage.getObject(asset.companyId, asset.objectKey, range ? { range } : undefined); const responseContentType = asset.contentType || object.contentType || "application/octet-stream"; const mediaType = responseContentType.split(";", 1)[0]?.trim().toLowerCase() ?? ""; const inlineSafe = mediaType !== SVG_CONTENT_TYPE && isInlineAttachmentContentType(mediaType); res.setHeader("Content-Type", responseContentType); - res.setHeader("Content-Length", String(asset.byteSize || object.contentLength || 0)); + res.setHeader("Content-Length", String(range ? range.end - range.start + 1 : asset.byteSize || object.contentLength || 0)); + if (range) { + res.status(206); + res.setHeader("Content-Range", `bytes ${range.start}-${range.end}/${asset.byteSize}`); + } res.setHeader("Cache-Control", "private, max-age=60"); res.setHeader("X-Content-Type-Options", "nosniff"); if (!inlineSafe) { @@ -349,6 +367,7 @@ export function assetRoutes(db: Db, storage: StorageService) { object.stream.on("error", (err) => { next(err); }); + res.on("close", () => object.stream.destroy()); object.stream.pipe(res); }); diff --git a/server/src/services/native-runtime/runner-api-client.ts b/server/src/services/native-runtime/runner-api-client.ts index 7fb6123e199..3895fb9e30a 100644 --- a/server/src/services/native-runtime/runner-api-client.ts +++ b/server/src/services/native-runtime/runner-api-client.ts @@ -149,6 +149,14 @@ export async function executeRunnerApi(input: RunnerApiCall, context: RunnerApiC const url = runnerApiUrl(operation, input, context, io.apiUrl); if (!io.token) throw new Error("Paperclip run authentication is unavailable"); const headers = new Headers({ Authorization: `Bearer ${io.token}`, "X-Paperclip-Run-Id": context.runId }); + const textOffset = input.responseText?.offsetBytes ?? 0; + const textLimit = input.responseText?.limitBytes ?? RUNNER_API_INLINE_BYTES; + const assetTextRange = input.responseText && operation.operationId === "GET /api/assets/{assetId}/content" + ? { start: Math.max(0, textOffset - 1), end: textOffset + textLimit } + : undefined; + // One byte before the requested offset permits an EOF read; one byte after + // the page lets the decoder trim an incomplete UTF-8 code point. + if (assetTextRange) headers.set("Range", `bytes=${assetTextRange.start}-${assetTextRange.end}`); let body: BodyInit | undefined; const contentType = input.contentType ?? (input.files?.length ? "multipart/form-data" : "application/json"); if (/\r|\n/.test(contentType)) throw badRequest("Invalid content type"); @@ -186,7 +194,7 @@ export async function executeRunnerApi(input: RunnerApiCall, context: RunnerApiC await io.beforeDispatch?.(); try { response = await (io.fetch ?? fetch)(url, { method: operation.method, headers, body, redirect: "manual", signal: AbortSignal.timeout(RUNNER_API_TIMEOUT_MS) }); - bytes = await readBoundedResponse(response); + bytes = await readBoundedResponse(response, assetTextRange && response.ok ? textLimit + 2 : RUNNER_API_MAX_BYTES); } catch { return { ok: false, status: null, operationId: operation.operationId, error: "api_transport_failure", outcome: ["GET", "HEAD", "OPTIONS"].includes(operation.method) ? "read_failed" : "unknown", guidance: "Inspect current state before retrying a mutation; it may already have succeeded." }; } @@ -200,17 +208,39 @@ export async function executeRunnerApi(input: RunnerApiCall, context: RunnerApiC if (response.status >= 300 && response.status < 400) return { ...base, ok: false, error: "api_redirect_not_followed" }; if (input.responseText) { if (!/json|^text\//i.test(type)) return { ...base, ok: false, error: "response_text_requires_text_content" }; - const offsetBytes = input.responseText.offsetBytes ?? 0; - if (offsetBytes > bytes.length || (offsetBytes < bytes.length && (bytes[offsetBytes] & 0xc0) === 0x80)) { - return { ...base, ok: false, error: "response_text_invalid_offset", byteSize: bytes.length }; + let totalBytes = bytes.length; + let baseOffset = 0; + if (assetTextRange && response.ok) { + const range = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(response.headers.get("content-range") ?? ""); + if (response.status === 200 && bytes.length === 0 && textOffset === 0) { + // An empty asset has no satisfiable byte range; its route returns 200. + } else if (response.status !== 206 || !range) { + return { ...base, ok: false, error: "response_text_invalid_range" }; + } else { + const [start, end, total] = range.slice(1).map(Number); + if (![start, end, total].every(Number.isSafeInteger) || start !== assetTextRange.start + || total > RUNNER_API_MAX_BYTES || total < 1 || end !== Math.min(assetTextRange.end, total - 1) + || end < start || bytes.length !== end - start + 1) { + return { ...base, ok: false, error: "response_text_invalid_range" }; + } + baseOffset = start; + totalBytes = total; + } } - let end = Math.min(bytes.length, offsetBytes + (input.responseText.limitBytes ?? RUNNER_API_INLINE_BYTES)); + // HTTP denials are evidence, not text pages of an authorized asset. + const offsetBytes = response.ok ? textOffset : 0; + const localOffset = offsetBytes - baseOffset; + if (offsetBytes > totalBytes || localOffset < 0 || (localOffset < bytes.length && (bytes[localOffset] & 0xc0) === 0x80)) { + return { ...base, ok: false, error: "response_text_invalid_offset", byteSize: totalBytes }; + } + let end = Math.min(bytes.length, localOffset + textLimit); // Do not cut a UTF-8 code point. The next offset always starts a whole one. - while (end < bytes.length && end > offsetBytes && (bytes[end] & 0xc0) === 0x80) end--; - if (end === offsetBytes && end < bytes.length) return { ...base, ok: false, error: "response_text_invalid_utf8" }; + while (end < bytes.length && end > localOffset && (bytes[end] & 0xc0) === 0x80) end--; + if (end === localOffset && end < bytes.length) return { ...base, ok: false, error: "response_text_invalid_utf8" }; try { - const data = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes.subarray(offsetBytes, end)); - return { ...base, data, responseText: { offsetBytes, nextOffsetBytes: end < bytes.length ? end : null, totalBytes: bytes.length } }; + const data = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes.subarray(localOffset, end)); + const nextOffset = baseOffset + end; + return { ...base, data, responseText: { offsetBytes, nextOffsetBytes: nextOffset < totalBytes ? nextOffset : null, totalBytes } }; } catch { return { ...base, ok: false, error: "response_text_invalid_utf8" }; } diff --git a/server/src/services/native-runtime/runner-api.integration.test.ts b/server/src/services/native-runtime/runner-api.integration.test.ts index f04b2654c6e..f3238e64f77 100644 --- a/server/src/services/native-runtime/runner-api.integration.test.ts +++ b/server/src/services/native-runtime/runner-api.integration.test.ts @@ -258,17 +258,26 @@ else if(m.id!==undefined) send({id:m.id,result:{}}); const initial = await call(fixture.authority, asset.id); expect(initial).toMatchObject({ status: 200, artifact: { byteSize: Buffer.byteLength(text) } }); const before = (await fixture.snapshot()).assets.length; + const reads = vi.spyOn(server.storage, "getObject"); let offsetBytes = 0; let result = ""; - do { - const page = await call(fixture.authority, initial.artifact.artifactId, { offsetBytes, limitBytes: 4096 }); - expect(page).toMatchObject({ status: 200, responseText: { offsetBytes, totalBytes: Buffer.byteLength(text) } }); - result += page.data; - offsetBytes = page.responseText.nextOffsetBytes; - } while (offsetBytes !== null); - expect(JSON.parse(result)).toEqual(JSON.parse(text)); - expect((await fixture.snapshot()).assets).toHaveLength(before); - expect(await call(foreign.authority, initial.artifact.artifactId, { offsetBytes: 0 })).toMatchObject({ ok: false, status: 404 }); + try { + do { + const page = await call(fixture.authority, initial.artifact.artifactId, { offsetBytes, limitBytes: 4096 }); + expect(page).toMatchObject({ status: 206, responseText: { offsetBytes, totalBytes: Buffer.byteLength(text) } }); + result += page.data; + offsetBytes = page.responseText.nextOffsetBytes; + } while (offsetBytes !== null); + expect(JSON.parse(result)).toEqual(JSON.parse(text)); + expect(reads.mock.calls.length).toBeGreaterThan(1); + expect(reads.mock.calls.every(([, , options]) => options?.range)).toBe(true); + const storageBytes = reads.mock.calls.reduce((total, [, , options]) => total + options!.range!.end - options!.range!.start + 1, 0); + expect(storageBytes).toBeLessThan(Buffer.byteLength(text) + reads.mock.calls.length * 5); + expect((await fixture.snapshot()).assets).toHaveLength(before); + expect(await call(foreign.authority, initial.artifact.artifactId, { offsetBytes: 0 })).toMatchObject({ ok: false, status: 404 }); + } finally { + reads.mockRestore(); + } }); it("contains workspace files, checks artifact ownership, and persists downloads", async () => { diff --git a/server/src/services/native-runtime/runner-api.test.ts b/server/src/services/native-runtime/runner-api.test.ts index 3c164d60946..7894a0602e6 100644 --- a/server/src/services/native-runtime/runner-api.test.ts +++ b/server/src/services/native-runtime/runner-api.test.ts @@ -608,6 +608,48 @@ describe("runner API request boundary", () => { expect(saveResponse).not.toHaveBeenCalled(); expect(request.mock.calls.every(([, options]) => new Headers(options?.headers).get("Authorization") === "Bearer private-agent-token")).toBe(true); }); + it("pages a ten MiB asset with linear transfer and UTF-8 boundaries", async () => { + const bytes = Buffer.from("🧭é".repeat(Math.floor(10 * 1024 * 1024 / 6))); + let transferred = 0; + const request = vi.fn(async (_url, init) => { + const range = new Headers(init?.headers).get("range"); + const match = /^bytes=(\d+)-(\d+)$/.exec(range ?? ""); + if (!match) { transferred += bytes.length; return new Response(bytes, { headers: { "content-type": "text/plain" } }); } + const start = Number(match[1]), end = Math.min(Number(match[2]), bytes.length - 1); + const part = bytes.subarray(start, end + 1); transferred += part.length; + return new Response(part, { status: 206, headers: { "content-type": "text/plain", "content-range": `bytes ${start}-${end}/${bytes.length}` } }); + }); + let offsetBytes: number | null = 0; + const parts: string[] = []; + do { + const page: any = await executeRunnerApi({ operationId: "GET /api/assets/{assetId}/content", pathParams: { assetId: "large" }, responseText: { offsetBytes } }, context, io(request)); + expect(page.ok).toBe(true); + parts.push(page.data); offsetBytes = page.responseText.nextOffsetBytes; + } while (offsetBytes !== null); + expect(parts.join("")).toBe(bytes.toString()); + expect(transferred).toBeLessThan(bytes.length + request.mock.calls.length * 5); + expect(request.mock.calls.every(([, init]) => new Headers(init?.headers).has("range"))).toBe(true); + }); + it.each([ + ["bytes 4-8/20", "12345"], // wrong start + ["bytes 3-7/20", "12345"], // early end + ["bytes 3-8/20", "12345"], // truncated body + ["bytes 3-8/10485761", "123456"], // total exceeds transfer policy + ["bytes 3-8/*", "123456"], // unknown total + [null, "123456"], + ])("rejects inconsistent partial asset receipts: %s", async (range, body) => { + const result = await executeRunnerApi({ operationId: "GET /api/assets/{assetId}/content", pathParams: { assetId: "saved" }, responseText: { offsetBytes: 4, limitBytes: 4 } }, context, io(async () => new Response(body, { status: 206, headers: { "content-type": "text/plain", ...(range ? { "content-range": range } : {}) } }))); + expect(result).toMatchObject({ ok: false, error: "response_text_invalid_range" }); + }); + it("accepts EOF and rejects an offset inside a UTF-8 code point in ranged assets", async () => { + const call = (offsetBytes: number, bytes: Buffer, range: string) => executeRunnerApi({ operationId: "GET /api/assets/{assetId}/content", pathParams: { assetId: "saved" }, responseText: { offsetBytes, limitBytes: 4 } }, context, io(async () => new Response(new Uint8Array(bytes), { status: 206, headers: { "content-type": "text/plain", "content-range": range } }))); + expect(await call(2, Buffer.from([0xa9]), "bytes 1-1/2")).toMatchObject({ ok: true, data: "", responseText: { nextOffsetBytes: null, totalBytes: 2 } }); + expect(await call(1, Buffer.from("é"), "bytes 0-1/2")).toMatchObject({ ok: false, error: "response_text_invalid_offset" }); + }); + it("does not silently accept an asset server that ignores the byte range", async () => { + const result = await executeRunnerApi({ operationId: "GET /api/assets/{assetId}/content", pathParams: { assetId: "saved" }, responseText: { limitBytes: 4 } }, context, io(async () => new Response("unbounded", { headers: { "content-type": "text/plain" } }))); + expect(result).toMatchObject({ ok: false, error: "api_transport_failure" }); + }); it.each([{ offsetBytes: -1 }, { limitBytes: 0 }, { limitBytes: 3 }, { limitBytes: 24577 }, { offsetBytes: 0.5 }])("rejects invalid text windows before dispatch: %j", async responseText => { const request = vi.fn(); await expect(executeRunnerApi({ operationId: projects, responseText }, context, io(request))).rejects.toThrow("Invalid call_api arguments");