From a354242865454f2e9ee0085db788656231a35521 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 12 Nov 2026 09:55:53 -0600 Subject: [PATCH] fix(storage): end every first-round part digest before a failed upload settles The first round's digests were awaited as one promise, which rejects at the first failure. A cancelled or failed upload could then finish its cleanup while other digests of the round still read the source file. Each part now waits until every digest of the round has ended. Co-Authored-By: Claude Fable 5.2 --- .../__tests__/storage-s3-multipart.test.ts | 23 +++++++++++++++++++ server/src/storage/s3-multipart.ts | 9 +++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/server/src/__tests__/storage-s3-multipart.test.ts b/server/src/__tests__/storage-s3-multipart.test.ts index f7b0690f2d0..c541abfa2fd 100644 --- a/server/src/__tests__/storage-s3-multipart.test.ts +++ b/server/src/__tests__/storage-s3-multipart.test.ts @@ -11,6 +11,7 @@ import path from "node:path"; import { createS3StorageProvider } from "../storage/s3-provider.js"; import { createStorageService } from "../storage/service.js"; import { STORAGE_FAILURE_CLEANUP_MS } from "../storage/failure-cleanup.js"; +import { fileCryptoPoolLoad } from "../storage/file-crypto-pool.js"; describe("streamed S3 multipart uploads", () => { it("delivers four slow part payloads concurrently before any part is drained", async () => { @@ -532,6 +533,28 @@ describe("file-backed S3 multipart uploads", () => { } finally { send.mockRestore(); client.destroy(); await fs.rm(source.dir, { recursive: true, force: true }); } }); + it.each(["cancelled", "removed"])("leaves no part digest running or waiting when the source is %s during the first round", async reason => { + const source = await sourceFile(FILE_PART_CONCURRENCY + 2); + const client = new S3Client({ region: "us-east-1" }), cancellation = new AbortController(); + let digestsAtCleanup: ReturnType | undefined; + const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => { + if (command instanceof CreateMultipartUploadCommand) { + // The digests of the first round start right after this reply. + if (reason === "cancelled") setImmediate(() => cancellation.abort(new Error("run cancelled"))); + else await fs.rm(source.file); + return { UploadId: "upload" }; + } + if (command instanceof AbortMultipartUploadCommand) { digestsAtCleanup = fileCryptoPoolLoad(); return {}; } + throw new Error("must not send a part or complete the upload"); + }); + try { + await expect(putS3MultipartFile(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", + contentLength: source.size, body: Buffer.alloc(0), sourceFile: source.file, signal: cancellation.signal })).rejects.toThrow(); + expect(digestsAtCleanup).toEqual({ running: 0, waiting: 0 }); + expect(fileCryptoPoolLoad()).toEqual({ running: 0, waiting: 0 }); + } finally { send.mockRestore(); client.destroy(); await fs.rm(source.dir, { recursive: true, force: true }); } + }); + it("refuses a source file whose size differs from the declared length before starting an upload", async () => { const source = await sourceFile(3); const client = new S3Client({ region: "us-east-1" }); diff --git a/server/src/storage/s3-multipart.ts b/server/src/storage/s3-multipart.ts index 40661424b3d..d782d360fb0 100644 --- a/server/src/storage/s3-multipart.ts +++ b/server/src/storage/s3-multipart.ts @@ -255,11 +255,14 @@ export async function putS3MultipartFile( // The first round of parts is digested before any of them is sent. They // leave together, as they did when this thread digested them side by side, // so a rate limit still answers one burst rather than a trickle of requests. - const firstRound = Promise.all(Array.from({ length: Math.min(FILE_PART_CONCURRENCY, count) }, (_, index) => partMd5(index))); - firstRound.catch(() => undefined); + // Every digest of the round has ended, also a failed or cancelled one, + // before a part goes on: none still reads the file when the upload settles. + const firstRound = Array.from({ length: Math.min(FILE_PART_CONCURRENCY, count) }, (_, index) => partMd5(index)); + const firstRoundEnded = Promise.allSettled(firstRound); const sendPart = async (index: number) => { const start = index * partSize, length = Math.min(partSize, input.contentLength - start), end = start + length - 1; - const contentMd5 = index < FILE_PART_CONCURRENCY ? (await firstRound)[index]! : await partMd5(index); + if (index < firstRound.length) await firstRoundEnded; + const contentMd5 = await (firstRound[index] ?? partMd5(index)); for (let failures = 0, throttles = 0; ;) { controller.signal.throwIfAborted(); const body = createReadStream(file, { start, end, highWaterMark: 64 * 1024 });