diff --git a/server/src/__tests__/storage-s3-multipart.test.ts b/server/src/__tests__/storage-s3-multipart.test.ts index f7b0690f2d0..c541abfa2fd 100644 --- a/server/src/__tests__/storage-s3-multipart.test.ts +++ b/server/src/__tests__/storage-s3-multipart.test.ts @@ -11,6 +11,7 @@ import path from "node:path"; import { createS3StorageProvider } from "../storage/s3-provider.js"; import { createStorageService } from "../storage/service.js"; import { STORAGE_FAILURE_CLEANUP_MS } from "../storage/failure-cleanup.js"; +import { fileCryptoPoolLoad } from "../storage/file-crypto-pool.js"; describe("streamed S3 multipart uploads", () => { it("delivers four slow part payloads concurrently before any part is drained", async () => { @@ -532,6 +533,28 @@ describe("file-backed S3 multipart uploads", () => { } finally { send.mockRestore(); client.destroy(); await fs.rm(source.dir, { recursive: true, force: true }); } }); + it.each(["cancelled", "removed"])("leaves no part digest running or waiting when the source is %s during the first round", async reason => { + const source = await sourceFile(FILE_PART_CONCURRENCY + 2); + const client = new S3Client({ region: "us-east-1" }), cancellation = new AbortController(); + let digestsAtCleanup: ReturnType | undefined; + const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => { + if (command instanceof CreateMultipartUploadCommand) { + // The digests of the first round start right after this reply. + if (reason === "cancelled") setImmediate(() => cancellation.abort(new Error("run cancelled"))); + else await fs.rm(source.file); + return { UploadId: "upload" }; + } + if (command instanceof AbortMultipartUploadCommand) { digestsAtCleanup = fileCryptoPoolLoad(); return {}; } + throw new Error("must not send a part or complete the upload"); + }); + try { + await expect(putS3MultipartFile(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", + contentLength: source.size, body: Buffer.alloc(0), sourceFile: source.file, signal: cancellation.signal })).rejects.toThrow(); + expect(digestsAtCleanup).toEqual({ running: 0, waiting: 0 }); + expect(fileCryptoPoolLoad()).toEqual({ running: 0, waiting: 0 }); + } finally { send.mockRestore(); client.destroy(); await fs.rm(source.dir, { recursive: true, force: true }); } + }); + it("refuses a source file whose size differs from the declared length before starting an upload", async () => { const source = await sourceFile(3); const client = new S3Client({ region: "us-east-1" }); diff --git a/server/src/storage/s3-multipart.ts b/server/src/storage/s3-multipart.ts index 40661424b3d..d782d360fb0 100644 --- a/server/src/storage/s3-multipart.ts +++ b/server/src/storage/s3-multipart.ts @@ -255,11 +255,14 @@ export async function putS3MultipartFile( // The first round of parts is digested before any of them is sent. They // leave together, as they did when this thread digested them side by side, // so a rate limit still answers one burst rather than a trickle of requests. - const firstRound = Promise.all(Array.from({ length: Math.min(FILE_PART_CONCURRENCY, count) }, (_, index) => partMd5(index))); - firstRound.catch(() => undefined); + // Every digest of the round has ended, also a failed or cancelled one, + // before a part goes on: none still reads the file when the upload settles. + const firstRound = Array.from({ length: Math.min(FILE_PART_CONCURRENCY, count) }, (_, index) => partMd5(index)); + const firstRoundEnded = Promise.allSettled(firstRound); const sendPart = async (index: number) => { const start = index * partSize, length = Math.min(partSize, input.contentLength - start), end = start + length - 1; - const contentMd5 = index < FILE_PART_CONCURRENCY ? (await firstRound)[index]! : await partMd5(index); + if (index < firstRound.length) await firstRoundEnded; + const contentMd5 = await (firstRound[index] ?? partMd5(index)); for (let failures = 0, throttles = 0; ;) { controller.signal.throwIfAborted(); const body = createReadStream(file, { start, end, highWaterMark: 64 * 1024 });