From fa96716bb5ad5beb66d39c44210c87c39cf7dcac Mon Sep 17 00:00:00 2001 From: paperclipbugbot Date: Fri, 20 Nov 2026 05:37:17 -0800 Subject: [PATCH] fix(storage): back off a throttled small upload that has no source file (#22404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The server saves many objects to S3-compatible storage: skill packages, attachments, recovery archives and manifests > - Object storage answers a burst with `503 SlowDown`. Most large and file-backed uploads already back off (#20165, #20313, #20772, #21010) > - A small upload with no source file still had almost no retry. The S3 SDK never resends a stream body, so a stream got one attempt. A Buffer got the SDK's three closely spaced attempts > - Hosted production and staging reported `SlowDown` on `put_object` from request handlers, for example right after a skill import fetched its files. The error continues on the current production release > - This pull request holds a small stream's bytes within the existing buffered-payload budget, and backs off a throttled small upload like a file upload > - The benefit is that a short request-rate limit no longer fails a small save ## Linked Issues or Issue Description Refs #20165, #20313, #20772, #21010 (the same backoff for other upload paths). **What happened?** A request handler saved a small object with `storage.putFile` and a stream body (for example a skill package archive), or with a small Buffer. Object storage answered `503 SlowDown` ("Please reduce your request rate."). The stream was not sent again. The Buffer was sent three times within about a second. The request failed with a 500. **Expected behavior** A request-rate limit slows the upload down. It does not fail the save. **Steps to reproduce** 1. Point the S3 provider at an endpoint that answers the first `PUT` with `503 SlowDown`. 2. Call `putFile` with a small `Readable` body and no source file. 3. On `master` it throws `SlowDown: Please reduce your request rate.` after one `PUT`. **Paperclip version or commit** `master` at `201abacd6e`. ## What Changed - `server/src/storage/s3-multipart.ts`: `withBufferedUploadBody` holds a small stream body within the shared buffered-payload budget that multipart already uses. It checks the declared length. `sendThrottledRequestWithRetry` uses the same jittered exponential backoff as file uploads. It resends only request-rate limits (SlowDown, 503, 429). - `server/src/storage/s3-provider.ts`: the small `putObject` path without a source file sends Buffer bodies and buffered stream bodies through that retry. When the budget is full, the stream is sent once, as before. - `server/src/__tests__/storage-s3-small-upload-throttle.test.ts` (new): a real S3 client against a local endpoint. ## Verification - `cd server && npx vitest run src/__tests__/storage-s3-small-upload-throttle.test.ts`: with the `master` source files, 3 of 4 cases fail (the stream and Buffer cases reject with `SlowDown`). With this change, 4 of 4 pass. - `cd server && npx vitest run src/__tests__/storage-* src/storage`: 17 files, 135 tests pass on top of `201abacd6e`, including `does not repeat a write that timed out`. - I did not run the full server typecheck locally. The sandbox ran out of memory. CI must cover it. ## Risks - Low. A small stream is now held in memory, at most 16 MiB, inside the existing 32 MiB process budget. If the budget is full, behavior is unchanged. - Only request-rate limits are resent. The store did not apply those writes. Other write failures keep their SDK handling, so a timed-out write is still not repeated. - A stream that ends early or runs long now fails before any request is sent. - Conditional publication puts (`putObjectConditional`) are unchanged. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, via Claude Code, with tool use (shell, file edit, test execution) and read-only Sentry reads. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes (no doc change is necessary: this is internal storage behavior) - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: paperclipbugbot <356415228+paperclipbugbot@users.noreply.github.com> --- .../storage-s3-small-upload-throttle.test.ts | 180 ++++++++++++++++++ server/src/storage/s3-multipart.ts | 33 +++- server/src/storage/s3-provider.ts | 13 +- 3 files changed, 219 insertions(+), 7 deletions(-) create mode 100644 server/src/__tests__/storage-s3-small-upload-throttle.test.ts diff --git a/server/src/__tests__/storage-s3-small-upload-throttle.test.ts b/server/src/__tests__/storage-s3-small-upload-throttle.test.ts new file mode 100644 index 00000000000..2109a1c0752 --- /dev/null +++ b/server/src/__tests__/storage-s3-small-upload-throttle.test.ts @@ -0,0 +1,180 @@ +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { Readable } from "node:stream"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; +import { createStorageService } from "../storage/service.js"; + +type Seen = { method: string; key: string; body: Buffer }; +const seen: Seen[] = []; +let respond: (req: IncomingMessage, res: ServerResponse, index: number) => void = (_req, res) => res.end(); + +const server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", chunk => chunks.push(chunk as Buffer)); + req.on("end", () => { + seen.push({ method: req.method ?? "", key: (req.url ?? "").split("?")[0]!.split("/").pop() ?? "", body: Buffer.concat(chunks) }); + respond(req, res, seen.length - 1); + }); +}); +let endpoint = ""; +const savedEnv = { ...process.env }; + +beforeAll(async () => { + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + process.env.AWS_ACCESS_KEY_ID = "fixture-access-key"; + process.env.AWS_SECRET_ACCESS_KEY = "fixture-secret-key"; + delete process.env.AWS_SESSION_TOKEN; + delete process.env.AWS_PROFILE; +}); + +afterAll(async () => { + process.env = savedEnv; + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); +}); + +beforeEach(() => { seen.length = 0; vi.spyOn(Math, "random").mockReturnValue(0); }); +afterEach(() => { vi.restoreAllMocks(); }); + +const SLOW_DOWN = ` +SlowDownPlease reduce your request rate.req-503`; + +function throttleFirstPuts(count: number) { + respond = (req, res, index) => { + if (req.method === "PUT" && index < count) { + res.writeHead(503, { "content-type": "application/xml" }); + res.end(SLOW_DOWN); + return; + } + res.writeHead(200, { etag: '"saved"' }); + res.end(); + }; +} + +function storage() { + return createStorageService(createS3StorageProvider({ bucket: "bucket", region: "us-east-1", endpoint, forcePathStyle: true })); +} + +const sha256 = "0".repeat(64); + +describe("S3 throttling of small uploads without a source file", () => { + it("resends a small streamed upload after SlowDown", async () => { + const content = Buffer.from("skill package archive bytes"); + throttleFirstPuts(1); + + await expect(storage().putFile({ companyId: "company", namespace: "skill-packages", objectKey: "company/skill-packages/files.tar.gz", + originalFilename: "files.tar.gz", contentType: "application/gzip", body: Readable.from([content]), + byteSize: content.length, sha256 })).resolves.toMatchObject({ byteSize: content.length }); + + expect(seen.map(entry => entry.method)).toEqual(["PUT", "PUT"]); + expect(seen[1]!.body.equals(content)).toBe(true); + }); + + it("keeps backing off a small buffered upload after the SDK's own attempts are throttled", async () => { + const content = Buffer.from('{"files":[]}'); + throttleFirstPuts(3); + + await expect(storage().putFile({ companyId: "company", namespace: "skill-packages", objectKey: "company/skill-packages/manifest.json", + originalFilename: "manifest.json", contentType: "application/json", body: content })) + .resolves.toMatchObject({ byteSize: content.length }); + + expect(seen.map(entry => entry.method)).toEqual(["PUT", "PUT", "PUT", "PUT"]); + expect(seen[3]!.body.equals(content)).toBe(true); + }); + + it("rejects a streamed upload that ends before its declared size without sending it", async () => { + throttleFirstPuts(0); + + await expect(storage().putFile({ companyId: "company", namespace: "skill-packages", objectKey: "company/skill-packages/files.tar.gz", + originalFilename: "files.tar.gz", contentType: "application/gzip", body: Readable.from([Buffer.from("short")]), + byteSize: 10, sha256 })).rejects.toThrow("S3 upload source was truncated"); + expect(seen.filter(entry => entry.method === "PUT")).toEqual([]); + }); + + it("does not resend a small upload the store refused for good", async () => { + respond = (_req, res) => { + res.writeHead(403, { "content-type": "application/xml" }); + res.end(`AccessDenieddenied`); + }; + + await expect(storage().putFile({ companyId: "company", namespace: "skill-packages", objectKey: "company/skill-packages/files.tar.gz", + originalFilename: "files.tar.gz", contentType: "application/gzip", body: Readable.from([Buffer.from("bytes")]), + byteSize: 5, sha256 })).rejects.toMatchObject({ name: "AccessDenied" }); + expect(seen.filter(entry => entry.method === "PUT")).toHaveLength(1); + }); +}); + +describe("S3 small streamed uploads while the buffer budget is full", () => { + const MiB = 1024 * 1024; + const slowDown = (res: ServerResponse) => { res.writeHead(503, { "content-type": "application/xml" }); res.end(SLOW_DOWN); }; + const ok = (res: ServerResponse) => { res.writeHead(200, { etag: '"saved"' }); res.end(); }; + const putStream = (key: string, content: Buffer, signal?: AbortSignal) => storage().putFile({ companyId: "company", + namespace: "skill-packages", objectKey: `company/skill-packages/${key}`, originalFilename: key, contentType: "application/gzip", + body: Readable.from([content]), byteSize: content.length, sha256, signal }); + const awsChunkedPayload = (body: Buffer) => { + const chunks: Buffer[] = []; + for (let at = 0; at < body.length;) { + const end = body.indexOf("\r\n", at); + const size = end < 0 ? Number.NaN : Number.parseInt(body.subarray(at, end).toString(), 16); + if (!Number.isInteger(size) || size < 0) return undefined; + if (size === 0) return Buffer.concat(chunks); + chunks.push(body.subarray(end + 2, end + 2 + size)); + at = end + 4 + size; + } + return undefined; + }; + const puts = (key: string) => seen.filter(entry => entry.method === "PUT" && entry.key === key); + + it("streams another small upload once, intact, and gets the budget back from a failed and a cancelled upload", async () => { + const held = new Map(); + let bothHeld!: () => void; + const holding = new Promise(resolve => { bothHeld = resolve; }); + respond = (req, res) => { + if (req.method === "PUT" && seen.at(-1)!.key.startsWith("held-")) { + held.set(seen.at(-1)!.key, res); + if (held.size === 2) bothHeld(); + return; + } + if (req.method === "PUT" && seen.at(-1)!.key === "throttled") return slowDown(res); + ok(res); + }; + const cancel = new AbortController(); + const failing = putStream("held-failing", Buffer.alloc(16 * MiB, 1)); + const cancelled = putStream("held-cancelled", Buffer.alloc(16 * MiB, 2), cancel.signal); + failing.catch(() => undefined); + cancelled.catch(() => undefined); + await holding; + + const content = Buffer.from("skill package archive bytes"); + await expect(putStream("fallback", content)).resolves.toMatchObject({ byteSize: content.length }); + expect(puts("fallback").length).toBe(1); + expect(awsChunkedPayload(puts("fallback")[0]!.body)?.equals(content)).toBe(true); + await expect(putStream("throttled", content)).rejects.toMatchObject({ name: "SlowDown" }); + expect(puts("throttled").length).toBe(1); + + const refused = held.get("held-failing")!; + refused.writeHead(403, { "content-type": "application/xml" }); + refused.end(`AccessDenieddenied`); + await expect(failing).rejects.toMatchObject({ name: "AccessDenied" }); + cancel.abort(new Error("upload cancelled")); + await expect(cancelled).rejects.toThrow(); + held.get("held-cancelled")!.destroy(); + + const throttledOnce = new Set(); + respond = (req, res) => { + const key = seen.at(-1)!.key; + if (req.method === "PUT" && !throttledOnce.has(key)) { throttledOnce.add(key); return slowDown(res); } + ok(res); + }; + const first = Buffer.alloc(16 * MiB, 3); + const second = Buffer.alloc(16 * MiB, 4); + const outcomes = await Promise.allSettled([putStream("after-first", first), putStream("after-second", second)]); + expect(outcomes.map(outcome => outcome.status === "fulfilled" ? "saved" : (outcome.reason as Error).name)).toEqual(["saved", "saved"]); + expect(puts("after-first").length).toBe(2); + expect(puts("after-second").length).toBe(2); + expect(puts("after-first")[1]!.body.equals(first)).toBe(true); + expect(puts("after-second")[1]!.body.equals(second)).toBe(true); + }); +}); diff --git a/server/src/storage/s3-multipart.ts b/server/src/storage/s3-multipart.ts index d782d360fb0..2bb83e16331 100644 --- a/server/src/storage/s3-multipart.ts +++ b/server/src/storage/s3-multipart.ts @@ -4,7 +4,7 @@ import { recordStorageBackoff, recordStoragePart, recordStorageThrottle, withSto import { once } from "node:events"; import { createReadStream } from "node:fs"; import { stat } from "node:fs/promises"; -import { PassThrough, Readable } from "node:stream"; +import { addAbortSignal, PassThrough, Readable } from "node:stream"; import { setTimeout as delay } from "node:timers/promises"; import type { PutObjectInput, PutObjectReceipt } from "./types.js"; import { withS3FailureDiagnostics } from "./s3-failure.js"; @@ -186,7 +186,15 @@ function isTransientFileRequestFailure(error: unknown) { || (typeof name === "string" && TRANSIENT_FILE_REQUEST_NAMES.has(name)); } -export async function sendFileRequestWithRetry(signal: AbortSignal | undefined, send: () => Promise): Promise { +export function sendFileRequestWithRetry(signal: AbortSignal | undefined, send: () => Promise): Promise { + return sendRequestWithRetry(signal, send, true); +} + +export function sendThrottledRequestWithRetry(signal: AbortSignal | undefined, send: () => Promise): Promise { + return sendRequestWithRetry(signal, send, false); +} + +async function sendRequestWithRetry(signal: AbortSignal | undefined, send: () => Promise, retryTransient: boolean): Promise { for (let failures = 0, throttles = 0; ;) { signal?.throwIfAborted(); try { @@ -202,13 +210,32 @@ export async function sendFileRequestWithRetry(signal: AbortSignal | undefine await delay(wait, undefined, { signal }); continue; } - if (!isTransientFileRequestFailure(error) || ++failures >= FILE_PART_ATTEMPTS) throw error; + if (!retryTransient || !isTransientFileRequestFailure(error) || ++failures >= FILE_PART_ATTEMPTS) throw error; recordStorageBackoff(failures * 500); await delay(failures * 500, undefined, { signal }); } } } +export async function withBufferedUploadBody(source: Readable, length: number, signal: AbortSignal | undefined, + send: (body: Buffer) => Promise): Promise { + if (bufferedPayloadBytes + length > bufferedPayloadLimit) return false; + bufferedPayloadBytes += length; + try { + const payload = Buffer.allocUnsafe(length); + let position = 0; + for await (const chunk of signal ? addAbortSignal(signal, source) : source) { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array); + if (position + bytes.length > length) throw new Error("S3 upload source exceeds its declared size"); + bytes.copy(payload, position); + position += bytes.length; + } + if (position !== length) throw new Error("S3 upload source was truncated"); + await send(payload); + return true; + } finally { bufferedPayloadBytes -= length; } +} + /** Base64 MD5 of an inclusive byte range, as the Content-MD5 header expects. Hashed off the server's thread. */ export async function fileRangeMd5(file: string, start: number, end: number, signal?: AbortSignal) { const digest = await digestFileOffThread({ file, start, end, algorithm: "md5" }, signal); diff --git a/server/src/storage/s3-provider.ts b/server/src/storage/s3-provider.ts index e2c3bab8559..51f0f37d72a 100644 --- a/server/src/storage/s3-provider.ts +++ b/server/src/storage/s3-provider.ts @@ -9,7 +9,8 @@ import { ListMultipartUploadsCommand, AbortMultipartUploadCommand, } from "@aws-sdk/client-s3"; -import { assertSourceFileSize, fileRangeMd5, putS3Multipart, putS3MultipartFile, sendFileRequestWithRetry } from "./s3-multipart.js"; +import { assertSourceFileSize, fileRangeMd5, putS3Multipart, putS3MultipartFile, sendFileRequestWithRetry, + sendThrottledRequestWithRetry, withBufferedUploadBody } from "./s3-multipart.js"; import { createReadStream } from "node:fs"; import { createHash } from "node:crypto"; import { recordS3BodyFailure, withS3FailureDiagnostics } from "./s3-failure.js"; @@ -271,17 +272,21 @@ export function createS3StorageProvider(config: S3ProviderConfig): StorageProvid await putS3Multipart(client, bucket, key, input); return; } - const send = () => withS3FailureDiagnostics("put_object", () => client.send( + const send = (body: Buffer | Readable) => withS3FailureDiagnostics("put_object", () => client.send( new PutObjectCommand({ Bucket: bucket, Key: key, - Body: input.body, + Body: body, ContentType: input.contentType, ContentLength: input.contentLength, }), { abortSignal: input.signal }, ), input.signal); - await (Buffer.isBuffer(input.body) ? sendThroughStorageOutage(send, input.signal) : send()); + const sendBytes = async (body: Buffer) => { + await sendThroughStorageOutage(() => sendThrottledRequestWithRetry(input.signal, () => send(body)), input.signal); + }; + if (Buffer.isBuffer(input.body)) return sendBytes(input.body); + if (!(await withBufferedUploadBody(input.body, input.contentLength, input.signal, sendBytes))) await send(input.body); }, async getObject(input): Promise {