diff --git a/server/src/__tests__/storage-s3-small-upload-throttle.test.ts b/server/src/__tests__/storage-s3-small-upload-throttle.test.ts new file mode 100644 index 00000000000..2109a1c0752 --- /dev/null +++ b/server/src/__tests__/storage-s3-small-upload-throttle.test.ts @@ -0,0 +1,180 @@ +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { Readable } from "node:stream"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; +import { createStorageService } from "../storage/service.js"; + +type Seen = { method: string; key: string; body: Buffer }; +const seen: Seen[] = []; +let respond: (req: IncomingMessage, res: ServerResponse, index: number) => void = (_req, res) => res.end(); + +const server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", chunk => chunks.push(chunk as Buffer)); + req.on("end", () => { + seen.push({ method: req.method ?? "", key: (req.url ?? "").split("?")[0]!.split("/").pop() ?? "", body: Buffer.concat(chunks) }); + respond(req, res, seen.length - 1); + }); +}); +let endpoint = ""; +const savedEnv = { ...process.env }; + +beforeAll(async () => { + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + process.env.AWS_ACCESS_KEY_ID = "fixture-access-key"; + process.env.AWS_SECRET_ACCESS_KEY = "fixture-secret-key"; + delete process.env.AWS_SESSION_TOKEN; + delete process.env.AWS_PROFILE; +}); + +afterAll(async () => { + process.env = savedEnv; + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); +}); + +beforeEach(() => { seen.length = 0; vi.spyOn(Math, "random").mockReturnValue(0); }); +afterEach(() => { vi.restoreAllMocks(); }); + +const SLOW_DOWN = ` +SlowDownPlease reduce your request rate.req-503`; + +function throttleFirstPuts(count: number) { + respond = (req, res, index) => { + if (req.method === "PUT" && index < count) { + res.writeHead(503, { "content-type": "application/xml" }); + res.end(SLOW_DOWN); + return; + } + res.writeHead(200, { etag: '"saved"' }); + res.end(); + }; +} + +function storage() { + return createStorageService(createS3StorageProvider({ bucket: "bucket", region: "us-east-1", endpoint, forcePathStyle: true })); +} + +const sha256 = "0".repeat(64); + +describe("S3 throttling of small uploads without a source file", () => { + it("resends a small streamed upload after SlowDown", async () => { + const content = Buffer.from("skill package archive bytes"); + throttleFirstPuts(1); + + await expect(storage().putFile({ companyId: "company", namespace: "skill-packages", objectKey: "company/skill-packages/files.tar.gz", + originalFilename: "files.tar.gz", contentType: "application/gzip", body: Readable.from([content]), + byteSize: content.length, sha256 })).resolves.toMatchObject({ byteSize: content.length }); + + expect(seen.map(entry => entry.method)).toEqual(["PUT", "PUT"]); + expect(seen[1]!.body.equals(content)).toBe(true); + }); + + it("keeps backing off a small buffered upload after the SDK's own attempts are throttled", async () => { + const content = Buffer.from('{"files":[]}'); + throttleFirstPuts(3); + + await expect(storage().putFile({ companyId: "company", namespace: "skill-packages", objectKey: "company/skill-packages/manifest.json", + originalFilename: "manifest.json", contentType: "application/json", body: content })) + .resolves.toMatchObject({ byteSize: content.length }); + + expect(seen.map(entry => entry.method)).toEqual(["PUT", "PUT", "PUT", "PUT"]); + expect(seen[3]!.body.equals(content)).toBe(true); + }); + + it("rejects a streamed upload that ends before its declared size without sending it", async () => { + throttleFirstPuts(0); + + await expect(storage().putFile({ companyId: "company", namespace: "skill-packages", objectKey: "company/skill-packages/files.tar.gz", + originalFilename: "files.tar.gz", contentType: "application/gzip", body: Readable.from([Buffer.from("short")]), + byteSize: 10, sha256 })).rejects.toThrow("S3 upload source was truncated"); + expect(seen.filter(entry => entry.method === "PUT")).toEqual([]); + }); + + it("does not resend a small upload the store refused for good", async () => { + respond = (_req, res) => { + res.writeHead(403, { "content-type": "application/xml" }); + res.end(`AccessDenieddenied`); + }; + + await expect(storage().putFile({ companyId: "company", namespace: "skill-packages", objectKey: "company/skill-packages/files.tar.gz", + originalFilename: "files.tar.gz", contentType: "application/gzip", body: Readable.from([Buffer.from("bytes")]), + byteSize: 5, sha256 })).rejects.toMatchObject({ name: "AccessDenied" }); + expect(seen.filter(entry => entry.method === "PUT")).toHaveLength(1); + }); +}); + +describe("S3 small streamed uploads while the buffer budget is full", () => { + const MiB = 1024 * 1024; + const slowDown = (res: ServerResponse) => { res.writeHead(503, { "content-type": "application/xml" }); res.end(SLOW_DOWN); }; + const ok = (res: ServerResponse) => { res.writeHead(200, { etag: '"saved"' }); res.end(); }; + const putStream = (key: string, content: Buffer, signal?: AbortSignal) => storage().putFile({ companyId: "company", + namespace: "skill-packages", objectKey: `company/skill-packages/${key}`, originalFilename: key, contentType: "application/gzip", + body: Readable.from([content]), byteSize: content.length, sha256, signal }); + const awsChunkedPayload = (body: Buffer) => { + const chunks: Buffer[] = []; + for (let at = 0; at < body.length;) { + const end = body.indexOf("\r\n", at); + const size = end < 0 ? Number.NaN : Number.parseInt(body.subarray(at, end).toString(), 16); + if (!Number.isInteger(size) || size < 0) return undefined; + if (size === 0) return Buffer.concat(chunks); + chunks.push(body.subarray(end + 2, end + 2 + size)); + at = end + 4 + size; + } + return undefined; + }; + const puts = (key: string) => seen.filter(entry => entry.method === "PUT" && entry.key === key); + + it("streams another small upload once, intact, and gets the budget back from a failed and a cancelled upload", async () => { + const held = new Map(); + let bothHeld!: () => void; + const holding = new Promise(resolve => { bothHeld = resolve; }); + respond = (req, res) => { + if (req.method === "PUT" && seen.at(-1)!.key.startsWith("held-")) { + held.set(seen.at(-1)!.key, res); + if (held.size === 2) bothHeld(); + return; + } + if (req.method === "PUT" && seen.at(-1)!.key === "throttled") return slowDown(res); + ok(res); + }; + const cancel = new AbortController(); + const failing = putStream("held-failing", Buffer.alloc(16 * MiB, 1)); + const cancelled = putStream("held-cancelled", Buffer.alloc(16 * MiB, 2), cancel.signal); + failing.catch(() => undefined); + cancelled.catch(() => undefined); + await holding; + + const content = Buffer.from("skill package archive bytes"); + await expect(putStream("fallback", content)).resolves.toMatchObject({ byteSize: content.length }); + expect(puts("fallback").length).toBe(1); + expect(awsChunkedPayload(puts("fallback")[0]!.body)?.equals(content)).toBe(true); + await expect(putStream("throttled", content)).rejects.toMatchObject({ name: "SlowDown" }); + expect(puts("throttled").length).toBe(1); + + const refused = held.get("held-failing")!; + refused.writeHead(403, { "content-type": "application/xml" }); + refused.end(`AccessDenieddenied`); + await expect(failing).rejects.toMatchObject({ name: "AccessDenied" }); + cancel.abort(new Error("upload cancelled")); + await expect(cancelled).rejects.toThrow(); + held.get("held-cancelled")!.destroy(); + + const throttledOnce = new Set(); + respond = (req, res) => { + const key = seen.at(-1)!.key; + if (req.method === "PUT" && !throttledOnce.has(key)) { throttledOnce.add(key); return slowDown(res); } + ok(res); + }; + const first = Buffer.alloc(16 * MiB, 3); + const second = Buffer.alloc(16 * MiB, 4); + const outcomes = await Promise.allSettled([putStream("after-first", first), putStream("after-second", second)]); + expect(outcomes.map(outcome => outcome.status === "fulfilled" ? "saved" : (outcome.reason as Error).name)).toEqual(["saved", "saved"]); + expect(puts("after-first").length).toBe(2); + expect(puts("after-second").length).toBe(2); + expect(puts("after-first")[1]!.body.equals(first)).toBe(true); + expect(puts("after-second")[1]!.body.equals(second)).toBe(true); + }); +}); diff --git a/server/src/storage/s3-multipart.ts b/server/src/storage/s3-multipart.ts index d782d360fb0..2bb83e16331 100644 --- a/server/src/storage/s3-multipart.ts +++ b/server/src/storage/s3-multipart.ts @@ -4,7 +4,7 @@ import { recordStorageBackoff, recordStoragePart, recordStorageThrottle, withSto import { once } from "node:events"; import { createReadStream } from "node:fs"; import { stat } from "node:fs/promises"; -import { PassThrough, Readable } from "node:stream"; +import { addAbortSignal, PassThrough, Readable } from "node:stream"; import { setTimeout as delay } from "node:timers/promises"; import type { PutObjectInput, PutObjectReceipt } from "./types.js"; import { withS3FailureDiagnostics } from "./s3-failure.js"; @@ -186,7 +186,15 @@ function isTransientFileRequestFailure(error: unknown) { || (typeof name === "string" && TRANSIENT_FILE_REQUEST_NAMES.has(name)); } -export async function sendFileRequestWithRetry(signal: AbortSignal | undefined, send: () => Promise): Promise { +export function sendFileRequestWithRetry(signal: AbortSignal | undefined, send: () => Promise): Promise { + return sendRequestWithRetry(signal, send, true); +} + +export function sendThrottledRequestWithRetry(signal: AbortSignal | undefined, send: () => Promise): Promise { + return sendRequestWithRetry(signal, send, false); +} + +async function sendRequestWithRetry(signal: AbortSignal | undefined, send: () => Promise, retryTransient: boolean): Promise { for (let failures = 0, throttles = 0; ;) { signal?.throwIfAborted(); try { @@ -202,13 +210,32 @@ export async function sendFileRequestWithRetry(signal: AbortSignal | undefine await delay(wait, undefined, { signal }); continue; } - if (!isTransientFileRequestFailure(error) || ++failures >= FILE_PART_ATTEMPTS) throw error; + if (!retryTransient || !isTransientFileRequestFailure(error) || ++failures >= FILE_PART_ATTEMPTS) throw error; recordStorageBackoff(failures * 500); await delay(failures * 500, undefined, { signal }); } } } +export async function withBufferedUploadBody(source: Readable, length: number, signal: AbortSignal | undefined, + send: (body: Buffer) => Promise): Promise { + if (bufferedPayloadBytes + length > bufferedPayloadLimit) return false; + bufferedPayloadBytes += length; + try { + const payload = Buffer.allocUnsafe(length); + let position = 0; + for await (const chunk of signal ? addAbortSignal(signal, source) : source) { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array); + if (position + bytes.length > length) throw new Error("S3 upload source exceeds its declared size"); + bytes.copy(payload, position); + position += bytes.length; + } + if (position !== length) throw new Error("S3 upload source was truncated"); + await send(payload); + return true; + } finally { bufferedPayloadBytes -= length; } +} + /** Base64 MD5 of an inclusive byte range, as the Content-MD5 header expects. Hashed off the server's thread. */ export async function fileRangeMd5(file: string, start: number, end: number, signal?: AbortSignal) { const digest = await digestFileOffThread({ file, start, end, algorithm: "md5" }, signal); diff --git a/server/src/storage/s3-provider.ts b/server/src/storage/s3-provider.ts index e2c3bab8559..51f0f37d72a 100644 --- a/server/src/storage/s3-provider.ts +++ b/server/src/storage/s3-provider.ts @@ -9,7 +9,8 @@ import { ListMultipartUploadsCommand, AbortMultipartUploadCommand, } from "@aws-sdk/client-s3"; -import { assertSourceFileSize, fileRangeMd5, putS3Multipart, putS3MultipartFile, sendFileRequestWithRetry } from "./s3-multipart.js"; +import { assertSourceFileSize, fileRangeMd5, putS3Multipart, putS3MultipartFile, sendFileRequestWithRetry, + sendThrottledRequestWithRetry, withBufferedUploadBody } from "./s3-multipart.js"; import { createReadStream } from "node:fs"; import { createHash } from "node:crypto"; import { recordS3BodyFailure, withS3FailureDiagnostics } from "./s3-failure.js"; @@ -271,17 +272,21 @@ export function createS3StorageProvider(config: S3ProviderConfig): StorageProvid await putS3Multipart(client, bucket, key, input); return; } - const send = () => withS3FailureDiagnostics("put_object", () => client.send( + const send = (body: Buffer | Readable) => withS3FailureDiagnostics("put_object", () => client.send( new PutObjectCommand({ Bucket: bucket, Key: key, - Body: input.body, + Body: body, ContentType: input.contentType, ContentLength: input.contentLength, }), { abortSignal: input.signal }, ), input.signal); - await (Buffer.isBuffer(input.body) ? sendThroughStorageOutage(send, input.signal) : send()); + const sendBytes = async (body: Buffer) => { + await sendThroughStorageOutage(() => sendThrottledRequestWithRetry(input.signal, () => send(body)), input.signal); + }; + if (Buffer.isBuffer(input.body)) return sendBytes(input.body); + if (!(await withBufferedUploadBody(input.body, input.contentLength, input.signal, sendBytes))) await send(input.body); }, async getObject(input): Promise {