diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 3a5b5aa1440..bcd6a197f55 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -2224,10 +2224,31 @@ S3 multipart uploads stream up to four parts concurrently. One producer reads the source in order, and bounded stream buffers preserve backpressure. Failed uploads cancel and settle their active requests before multipart cleanup. Snapshot publication still requires full object hash and size verification. -Local snapshot hashing, encryption, and decryption use bounded 1 MiB filesystem -stream buffers to reduce event-loop handoffs under background load. Each stream -retains its existing integrity and cancellation checks; archive contents and -saved snapshot formats are unchanged. Reproduce the scheduling effect with: +Snapshot hashing, encryption, and decryption run on worker threads +(`server/src/storage/file-crypto-pool.ts`), so they do not hold the server's +thread. The Content-MD5 of each uploaded part is computed the same way. At most +two jobs run at once, each through two reused 1 MiB buffers. A waiting job holds +file names, not file contents, and a cancelled job stops at its next chunk. +Every integrity check runs as before, with the same failures; archive contents +and saved snapshot formats are unchanged. Reproduce the effect on the server's +thread with: + +```sh +pnpm --filter @paperclipai/server exec tsx scripts/benchmark-workspace-snapshot-crypto.ts --mib=1024 +``` + +The benchmark saves and restores one synthetic archive through the real storage +functions and the real S3 client. The object store is a local process that +checks each part's Content-MD5. A third process requests a trivial route from +the measured server and times each reply with its own clock. It reports +main-thread busy time, the longest event-loop delay, process CPU time, wall +time, peak memory, and the latency of the unrelated requests. To compare two +revisions, run it from a checkout of each. The result is synthetic: the store +is on loopback without TLS, and sandbox transfer is excluded. + +Archive transfers that still stream on the server's thread use bounded 1 MiB +filesystem stream buffers to reduce event-loop handoffs under background load. +Reproduce that scheduling effect with: ```sh node --expose-gc scripts/benchmark-workspace-archive-io.mjs diff --git a/doc/run-log-events.md b/doc/run-log-events.md index fd82824c201..0e1f3a0f2a5 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -698,16 +698,16 @@ run phases: | Phase | Step | | --- | --- | -| `storage.snapshot.download` | Get the encrypted archive from storage and hash it while it streams. | -| `storage.snapshot.decrypt` | Decrypt the archive to private scratch. | -| `storage.snapshot.verify_plaintext` | Hash the decrypted archive against its sealed digest. | -| `storage.snapshot.publish` | Copy the verified archive to its destination. | +| `storage.snapshot.download` | Get the encrypted archive from storage. | +| `storage.snapshot.decrypt` | Hash the downloaded archive and decrypt it to private scratch, at the same time on worker threads. | +| `storage.snapshot.verify_plaintext` | Hash the decrypted archive against its sealed digest while copying it beside its destination, on a worker thread. | +| `storage.snapshot.publish` | Link the verified copy to its destination. | | `workspace.snapshot.sandbox_restore` | Upload and unpack the archive in the sandbox. | | `workspace.snapshot.sandbox_upload` | Inside `sandbox_restore`: copy the archive into the sandbox. | | `workspace.snapshot.sandbox_extract` | Inside `sandbox_restore`: verify and unpack the archive, then publish the restored folder. | | `storage.snapshot.prior_attempts` | Check whether an earlier upload attempt already landed. | -| `storage.snapshot.encrypt` | Hash and encrypt the archive in one pass. | -| `storage.snapshot.digest` | Hash the encrypted file that is uploaded. | +| `storage.snapshot.encrypt` | Hash the archive, encrypt it and hash the encrypted file, in one pass on a worker thread. | +| `storage.snapshot.digest` | Record the digest of the encrypted file. The `encrypt` pass computes it, so this step takes no time. | | `storage.snapshot.put` | Upload the encrypted archive (multipart above 16 MiB). | | `storage.snapshot.verify` | Confirm the stored object (its size when the backend checked every part, otherwise a full read-back). | diff --git a/server/scripts/benchmark-workspace-snapshot-crypto.ts b/server/scripts/benchmark-workspace-snapshot-crypto.ts new file mode 100644 index 00000000000..2fb2d92927f --- /dev/null +++ b/server/scripts/benchmark-workspace-snapshot-crypto.ts @@ -0,0 +1,272 @@ +/** + * Measures what saving and restoring one workspace snapshot costs the server's + * main thread, and how long an unrelated request waits meanwhile. + * + * Three processes, so a blocked server thread cannot hide its own stall: + * + * server runs the real `uploadWorkspaceSnapshot` and + * `loadWorkspaceSnapshotArchive` against the real S3 provider, and + * answers a trivial `GET /` on the same thread. + * store an S3-compatible object store on loopback that checks every part's + * Content-MD5, as the hosted backend does. + * probe requests `GET /` from the server one after another and times each + * reply with its own clock. + * + * The archive is synthetic random bytes: the storage layer treats it as opaque. + * + * pnpm --filter @paperclipai/server exec tsx scripts/benchmark-workspace-snapshot-crypto.ts --mib=200 + * pnpm --filter @paperclipai/server exec tsx scripts/benchmark-workspace-snapshot-crypto.ts --mib=1024 + * + * Options: --mib=, --samples=, + * --no-load (skip the restore). To compare two revisions, run the same script + * from a checkout of each. Results vary with machine load: report it. + */ +import assert from "node:assert/strict"; +import { fork, type ChildProcess } from "node:child_process"; +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { createReadStream, createWriteStream } from "node:fs"; +import fs from "node:fs/promises"; +import { Agent, createServer, get, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { monitorEventLoopDelay, performance } from "node:perf_hooks"; +import { pipeline } from "node:stream/promises"; +import { setTimeout as delay } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; + +const filename = fileURLToPath(import.meta.url); +const option = (name: string, fallback: string) => process.argv.find(arg => arg.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback; +const role = option("role", "run"); +const MIB = 1024 * 1024; +const round = (value: number) => Math.round(value * 10) / 10; +const percentile = (sorted: number[], fraction: number) => sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)] ?? 0; +const listen = (server: ReturnType) => new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve((server.address() as AddressInfo).port)); +}); +const message = (child: ChildProcess | NodeJS.Process, kind: string) => new Promise((resolve, reject) => { + const onMessage = (value: { kind: string }) => { + if (value.kind === "failed") { child.off("message", onMessage); reject(new Error((value as unknown as { error: string }).error)); } + if (value.kind !== kind) return; + child.off("message", onMessage); resolve(value as T); + }; + child.on("message", onMessage); +}); + +type Phase = "idle" | "upload" | "load"; +type PhaseResult = { + wallMs: number; mainThreadBusyMs: number; mainThreadCpuMs: number | null; processCpuMs: number; + eventLoopDelayMaxMs: number; eventLoopDelayP99Ms: number; peakRssMib: number; steps: Record; +}; +type ProbeResult = Record; + +if (role === "store") await store(); +else if (role === "probe") await probe(); +else if (role === "server") await server(); +else await run(); + +/** S3-compatible store: multipart uploads with Content-MD5 checks, HEAD, ranged GET and DELETE. */ +async function store() { + const root = option("root", ""); + const objects = new Map; size: number; etag: string }>(); + const uploads = new Map>(); + const receive = async (request: IncomingMessage, file: string) => { + const digest = createHash("md5"); let size = 0; + const output = createWriteStream(file); + request.on("data", (chunk: Buffer) => { digest.update(chunk); size += chunk.length; }); + await pipeline(request, output); + return { size, md5: digest.digest() }; + }; + const text = async (request: IncomingMessage) => { let body = ""; for await (const chunk of request) body += chunk; return body; }; + const reject = (response: ServerResponse, status: number, code: string) => response.writeHead(status, { "content-type": "application/xml" }) + .end(`${code}${code}`); + const handle = async (request: IncomingMessage, response: ServerResponse) => { + const url = new URL(request.url!, "http://store"), key = url.pathname, uploadId = url.searchParams.get("uploadId"); + if (request.method === "POST" && url.searchParams.has("uploads")) { + const id = randomUUID(); uploads.set(id, new Map()); + response.end(`${id}`); return; + } + if (request.method === "PUT") { + const number = Number(url.searchParams.get("partNumber") ?? 1), file = path.join(root, `${randomUUID()}.part`); + const received = await receive(request, file), expected = request.headers["content-md5"]; + if (!expected || received.md5.toString("base64") !== expected || received.size !== Number(request.headers["content-length"])) { + await fs.rm(file, { force: true }); reject(response, 400, "BadDigest"); return; + } + const etag = `"${received.md5.toString("hex")}"`; + if (uploadId) uploads.get(uploadId)!.set(number, { file, size: received.size, etag }); + else objects.set(key, { parts: [{ file, size: received.size }], size: received.size, etag }); + response.writeHead(200, { etag }).end(); return; + } + if (request.method === "POST" && uploadId) { + const body = await text(request), saved = uploads.get(uploadId)!; + const numbers = [...body.matchAll(/(\d+)<\/PartNumber>/g)].map(match => Number(match[1])); + const parts = numbers.map(number => saved.get(number)!); + if (parts.some(part => !part) || numbers.length !== saved.size) { reject(response, 400, "InvalidPart"); return; } + uploads.delete(uploadId); + objects.set(key, { parts, size: parts.reduce((total, part) => total + part.size, 0), etag: `"${uploadId}"` }); + response.end("complete"); return; + } + if (request.method === "DELETE") { + for (const part of [...(uploadId ? uploads.get(uploadId)?.values() ?? [] : []), ...(uploadId ? [] : objects.get(key)?.parts ?? [])]) await fs.rm(part.file, { force: true }); + if (uploadId) uploads.delete(uploadId); else objects.delete(key); + response.writeHead(204).end(); return; + } + const object = objects.get(key); + if (!object) { reject(response, 404, request.method === "HEAD" ? "NotFound" : "NoSuchKey"); return; } + const range = /^bytes=(\d+)-(\d+)$/.exec(request.headers.range ?? ""); + const start = range ? Number(range[1]) : 0, end = range ? Math.min(Number(range[2]), object.size - 1) : object.size - 1; + const headers = { "content-length": String(end - start + 1), etag: object.etag, "last-modified": new Date().toUTCString(), + ...(range ? { "content-range": `bytes ${start}-${end}/${object.size}` } : {}) }; + response.writeHead(range ? 206 : 200, headers); response.setMaxListeners(0); + if (request.method === "HEAD") { response.end(); return; } + let offset = 0; + for (const part of object.parts) { + const from = Math.max(start, offset), to = Math.min(end, offset + part.size - 1); + if (from <= to) await pipeline(createReadStream(part.file, { start: from - offset, end: to - offset }), response, { end: false }); + offset += part.size; + } + response.end(); + }; + const http = createServer((request, response) => { handle(request, response).catch(() => response.destroy()); }); + process.send!({ kind: "ready", port: await listen(http) }); + process.on("message", () => { http.closeAllConnections(); http.close(() => process.disconnect()); }); +} + +/** Sequential trivial requests to the server, bucketed by the phase the server reports. */ +async function probe() { + const port = Number(option("port", "0")), agent = new Agent({ keepAlive: true, maxSockets: 1 }); + const samples: Record = { idle: [], upload: [], load: [] }; + let active = true; + process.on("message", () => { active = false; }); + process.send!({ kind: "ready" }); + while (active) { + const started = performance.now(); + const phase = await new Promise(resolve => { + const request = get({ host: "127.0.0.1", port, agent, path: "/" }, response => { + response.resume(); response.once("end", () => resolve(response.headers["x-phase"] as Phase)); + }); + request.once("error", () => resolve(null)); + }); + if (phase) samples[phase].push(performance.now() - started); + await delay(2); + } + agent.destroy(); + const summary = Object.fromEntries(Object.entries(samples).map(([phase, values]) => { + const sorted = values.sort((a, b) => a - b); + return [phase, { requests: sorted.length, p50Ms: round(percentile(sorted, 0.5)), p99Ms: round(percentile(sorted, 0.99)), + maxMs: round(sorted.at(-1) ?? 0), over20Ms: sorted.filter(value => value > 20).length, over100Ms: sorted.filter(value => value > 100).length }]; + })); + process.send!({ kind: "result", summary }); + process.disconnect(); +} + +/** The process under measurement. */ +async function server() { + const root = option("root", ""), archivePath = option("archive", ""), restore = option("load", "1") === "1"; + process.env.PAPERCLIP_SECRETS_MASTER_KEY ??= randomBytes(32).toString("hex"); + process.env.AWS_ACCESS_KEY_ID = "benchmark"; process.env.AWS_SECRET_ACCESS_KEY = "benchmark"; process.env.AWS_SESSION_TOKEN = ""; + const { createDb, closeRegisteredClients } = await import("../../packages/db/src/index.js"); + const { createS3StorageProvider } = await import("../src/storage/s3-provider.js"); + const { createStorageService } = await import("../src/storage/service.js"); + const snapshots = await import("../src/services/remote-workspace-snapshot-storage.js"); + const databaseUrl = option("database", ""), db = createDb(databaseUrl); + const storage = createStorageService(createS3StorageProvider({ bucket: "bucket", region: "us-east-1", forcePathStyle: true, + endpoint: `http://127.0.0.1:${option("store", "0")}` })); + let phase: Phase = "idle"; + const http = createServer((_request, response) => { response.setHeader("x-phase", phase); response.end("ok"); }); + const histogram = monitorEventLoopDelay({ resolution: 1 }); histogram.enable(); + const threadCpu = (process as unknown as { threadCpuUsage?: (previous?: NodeJS.CpuUsage) => NodeJS.CpuUsage }).threadCpuUsage?.bind(process); + const measure = async (name: Exclude, work: (timing: ReturnType) => Promise): Promise => { + const timing = snapshots.newWorkspaceSnapshotStorageTiming(); + let peakRss = process.memoryUsage.rss(); + const sampler = setInterval(() => { peakRss = Math.max(peakRss, process.memoryUsage.rss()); }, 10); + await delay(50); histogram.reset(); + const utilization = performance.eventLoopUtilization(), cpu = process.cpuUsage(), thread = threadCpu?.(), started = performance.now(); + phase = name; + try { await work(timing); } + finally { phase = "idle"; } + const wallMs = performance.now() - started, busy = performance.eventLoopUtilization(utilization); + const used = process.cpuUsage(cpu), main = thread ? threadCpu!(thread) : null; + clearInterval(sampler); + const { requests: _requests, ...steps } = timing; + return { wallMs: round(wallMs), mainThreadBusyMs: round(busy.active), mainThreadCpuMs: main ? round((main.user + main.system) / 1000) : null, + processCpuMs: round((used.user + used.system) / 1000), eventLoopDelayMaxMs: round(histogram.max / 1e6), + eventLoopDelayP99Ms: round(histogram.percentile(99) / 1e6), peakRssMib: round(peakRss / MIB), steps }; + }; + const save = async (file: string, timing?: ReturnType) => { + const companyId = randomUUID(), workspaceId = randomUUID(); + const row = await snapshots.beginWorkspaceSnapshot(db, { companyId, workspaceId, operationId: randomUUID(), generation: 1, epoch: 0 }); + return snapshots.uploadWorkspaceSnapshot({ db, storage, snapshotId: row.id, companyId, workspaceId, archivePath: file, + manifestDigest: "ab".repeat(32), timing }); + }; + const sha256 = async (file: string) => { + const digest = createHash("sha256"); + for await (const chunk of createReadStream(file, { highWaterMark: MIB })) digest.update(chunk); + return digest.digest("hex"); + }; + process.send!({ kind: "ready", port: await listen(http) }); + process.on("message", async (request: { kind: string }) => { + try { + if (request.kind === "close") { + await closeRegisteredClients(databaseUrl); + http.closeAllConnections(); http.close(() => process.disconnect()); + return; + } + // One small save and restore first, so module loading, the database + // connection and the storage client are not counted. + const warm = path.join(root, "warm.pcws"); await fs.writeFile(warm, randomBytes(4 * MIB)); + await snapshots.loadWorkspaceSnapshotArchive({ storage, reference: await save(warm), targetPath: path.join(root, `warm-${randomUUID()}.out`) }); + let reference!: Awaited>; + const upload = await measure("upload", async timing => { reference = await save(archivePath, timing); }); + const restored = path.join(root, `restored-${randomUUID()}.pcws`); + const load = restore ? await measure("load", timing => snapshots.loadWorkspaceSnapshotArchive({ storage, reference, targetPath: restored, timing })) : null; + if (restore) { assert.equal(await sha256(restored), await sha256(archivePath), "restored archive differs from the saved one"); await fs.rm(restored); } + process.send!({ kind: "result", upload, load, storedBytes: reference.byteSize }); + } catch (error) { process.send!({ kind: "failed", error: error instanceof Error ? error.stack ?? error.message : String(error) }); } + }); +} + +async function run() { + const mib = Number(option("mib", "200")), samples = Number(option("samples", "1")), restore = !process.argv.includes("--no-load"); + assert.ok(Number.isSafeInteger(mib) && mib >= 17 && mib <= 8192, "--mib must be between 17 and 8192"); + assert.ok(Number.isSafeInteger(samples) && samples >= 1 && samples <= 5, "--samples must be between 1 and 5"); + const { startEmbeddedPostgresTestDatabase } = await import("../../packages/db/src/index.js"); + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "pc-snapshot-crypto-benchmark-"))); + const children: ChildProcess[] = []; + const start = async (name: string, args: string[]) => { + const child = fork(filename, [`--role=${name}`, ...args], { stdio: ["ignore", "inherit", "inherit", "ipc"], + env: { ...process.env, TMPDIR: path.join(root, "tmp") } }); + children.push(child); + return { child, ready: await message(child, "ready") }; + }; + let database: Awaited> | undefined; + try { + await fs.mkdir(path.join(root, "tmp")); await fs.mkdir(path.join(root, "store")); await fs.mkdir(path.join(root, "work")); + const archive = path.join(root, "archive.pcws"), handle = await fs.open(archive, "wx", 0o600), block = randomBytes(MIB); + try { for (let index = 0; index < mib; index++) { block.writeUInt32BE(index, 0); await handle.writeFile(block); } } finally { await handle.close(); } + database = await startEmbeddedPostgresTestDatabase("snapshot-crypto-benchmark-"); + const objectStore = await start<{ port: number }>("store", [`--root=${path.join(root, "store")}`]); + const results = []; + for (let sample = 0; sample < samples; sample++) { + const subject = await start<{ port: number }>("server", [`--root=${path.join(root, "work")}`, `--archive=${archive}`, + `--store=${objectStore.ready.port}`, `--database=${database.connectionString}`, `--load=${restore ? 1 : 0}`]); + const prober = await start("probe", [`--port=${subject.ready.port}`]); + const loadAverage = round(os.loadavg()[0]!); + subject.child.send({ kind: "run" }); + const measured = await message<{ upload: PhaseResult; load: PhaseResult | null; storedBytes: number }>(subject.child, "result"); + prober.child.send({ kind: "stop" }); + const probed = (await message<{ summary: ProbeResult }>(prober.child, "result")).summary; + subject.child.send({ kind: "close" }); + results.push({ sample, archiveMib: mib, storedBytes: measured.storedBytes, loadAverage, + upload: { ...measured.upload, unrelatedRequest: probed.upload }, + ...(measured.load ? { load: { ...measured.load, unrelatedRequest: probed.load } } : {}) }); + } + objectStore.child.send({ kind: "close" }); + console.log(JSON.stringify({ node: process.version, cpus: os.cpus().length, results }, null, 2)); + } finally { + for (const child of children) if (child.connected) child.kill(); + await database?.cleanup(); + await fs.rm(root, { recursive: true, force: true }); + } +} diff --git a/server/src/__tests__/storage-s3-multipart.test.ts b/server/src/__tests__/storage-s3-multipart.test.ts index f7b0690f2d0..c541abfa2fd 100644 --- a/server/src/__tests__/storage-s3-multipart.test.ts +++ b/server/src/__tests__/storage-s3-multipart.test.ts @@ -11,6 +11,7 @@ import path from "node:path"; import { createS3StorageProvider } from "../storage/s3-provider.js"; import { createStorageService } from "../storage/service.js"; import { STORAGE_FAILURE_CLEANUP_MS } from "../storage/failure-cleanup.js"; +import { fileCryptoPoolLoad } from "../storage/file-crypto-pool.js"; describe("streamed S3 multipart uploads", () => { it("delivers four slow part payloads concurrently before any part is drained", async () => { @@ -532,6 +533,28 @@ describe("file-backed S3 multipart uploads", () => { } finally { send.mockRestore(); client.destroy(); await fs.rm(source.dir, { recursive: true, force: true }); } }); + it.each(["cancelled", "removed"])("leaves no part digest running or waiting when the source is %s during the first round", async reason => { + const source = await sourceFile(FILE_PART_CONCURRENCY + 2); + const client = new S3Client({ region: "us-east-1" }), cancellation = new AbortController(); + let digestsAtCleanup: ReturnType | undefined; + const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => { + if (command instanceof CreateMultipartUploadCommand) { + // The digests of the first round start right after this reply. + if (reason === "cancelled") setImmediate(() => cancellation.abort(new Error("run cancelled"))); + else await fs.rm(source.file); + return { UploadId: "upload" }; + } + if (command instanceof AbortMultipartUploadCommand) { digestsAtCleanup = fileCryptoPoolLoad(); return {}; } + throw new Error("must not send a part or complete the upload"); + }); + try { + await expect(putS3MultipartFile(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", + contentLength: source.size, body: Buffer.alloc(0), sourceFile: source.file, signal: cancellation.signal })).rejects.toThrow(); + expect(digestsAtCleanup).toEqual({ running: 0, waiting: 0 }); + expect(fileCryptoPoolLoad()).toEqual({ running: 0, waiting: 0 }); + } finally { send.mockRestore(); client.destroy(); await fs.rm(source.dir, { recursive: true, force: true }); } + }); + it("refuses a source file whose size differs from the declared length before starting an upload", async () => { const source = await sourceFile(3); const client = new S3Client({ region: "us-east-1" }); diff --git a/server/src/services/remote-workspace-snapshot-storage.test.ts b/server/src/services/remote-workspace-snapshot-storage.test.ts index d3eab02d39a..b2aa609e849 100644 --- a/server/src/services/remote-workspace-snapshot-storage.test.ts +++ b/server/src/services/remote-workspace-snapshot-storage.test.ts @@ -1,4 +1,5 @@ -import { randomBytes, randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -6,7 +7,7 @@ import { createServer, type ServerResponse } from "node:http"; import type { AddressInfo } from "node:net"; import { setTimeout as wait } from "node:timers/promises"; import { Readable } from "node:stream"; -import { appendFileSync, readdirSync } from "node:fs"; +import { appendFileSync, closeSync, openSync, readdirSync, statSync, truncateSync, writeSync } from "node:fs"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; import { applyPendingMigrations, closeRegisteredClients, createDb, executionWorkspaceSnapshots } from "@paperclipai/db"; @@ -464,6 +465,158 @@ suite("remote workspace snapshot storage", () => { await expect(loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath })).rejects.toMatchObject({ code: "EEXIST" }); expect(await fs.readFile(targetPath, "utf8")).toBe("existing"); }); + // The digest and cipher passes of a save and a restore run on worker threads, + // so they neither wait for the server's thread nor hold it. + const MIB = 1024 * 1024; + /** Counts the bytes this thread digests, encrypts or decrypts. */ + const serverThreadCrypto = () => { + const key = randomBytes(32), iv = randomBytes(12); + const spies = [createHash("sha256"), createCipheriv("aes-256-gcm", key, iv), createDecipheriv("aes-256-gcm", key, iv)] + .map(instance => vi.spyOn(Object.getPrototypeOf(instance) as { update(data: string | Uint8Array): unknown }, "update")); + return { + bytes: () => spies.reduce((total, spy) => total + spy.mock.calls.reduce((sum, [data]) => sum + data.length, 0), 0), + restore: () => { for (const spy of spies) spy.mockRestore(); }, + }; + }; + /** Sends the save's private scratch under `root`, and finds its encrypted file once encryption has begun. */ + const encryptionScratch = async () => { + const scratchRoot = path.join(root, "encrypt-scratch"); await fs.mkdir(scratchRoot); + const temp = vi.spyOn(os, "tmpdir").mockReturnValue(scratchRoot); + const started = () => vi.waitFor(async () => { + const file = path.join(scratchRoot, (await fs.readdir(scratchRoot))[0] ?? "missing", "archive.encrypted"); + await fs.access(file); + return file; + }, { timeout: 30_000, interval: 5 }); + return { scratchRoot, started, restore: () => temp.mockRestore() }; + }; + const storedFiles = async () => (await fs.readdir(path.join(root, "objects"), { recursive: true, withFileTypes: true }) + .catch((error: NodeJS.ErrnoException) => { if (error.code !== "ENOENT") throw error; return []; })).filter(entry => entry.isFile()); + const stagedPlaintext = async () => (await fs.readdir(root)).filter(name => name.startsWith(".workspace-snapshot-")); + const rewrap = async (ref: Awaited>, change: Record) => { + const wrapped = JSON.parse(await localEncryptedProvider.resolveVersion({ material: ref.key, externalRef: null })); + return { ...ref, key: (await localEncryptedProvider.createVersion({ value: JSON.stringify({ ...wrapped, ...change }) })).material }; + }; + + it("keeps encrypting a save while the server thread is occupied by other work", async () => { + const f = await fixture(randomBytes(24 * MIB)), scratch = await encryptionScratch(); + try { + const saving = uploadWorkspaceSnapshot(f.upload); + saving.catch(() => undefined); + const encrypted = await scratch.started(); + // Nothing else on this thread runs until this loop ends: no timer, no + // I/O callback, no promise. The encrypted file has to complete anyway. + const complete = f.captured.byteSize + 7 + 12 + 16, deadline = Date.now() + 30_000, pause = new Int32Array(new SharedArrayBuffer(4)); + let size = 0; + while ((size = statSync(encrypted).size) < complete && Date.now() < deadline) Atomics.wait(pause, 0, 0, 5); + expect(size).toBe(complete); + const ref = await saving, restored = path.join(root, "restored-after-busy.pcws"); + await loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath: restored }); + expect((await fs.readFile(restored)).equals(await fs.readFile(f.archivePath))).toBe(true); + } finally { scratch.restore(); } + }, 90_000); + it.skipIf(process.platform === "win32")("answers unrelated requests while a save is being encrypted, and digests or encrypts none of it on the server thread", async () => { + const f = await fixture(), archivePath = path.join(root, "streamed.pcws"), scratch = await encryptionScratch(); + // A named pipe keeps the save encrypting for exactly as long as this test + // feeds it, so "in progress" is a fact here and not a matter of timing. + execFileSync("mkfifo", [archivePath]); + const controller = new AbortController(), feed = await fs.open(archivePath, "r+"), onThread = serverThreadCrypto(); + let settled = false; + const saving = uploadWorkspaceSnapshot({ ...f.upload, archivePath, signal: controller.signal }) + .then(() => "saved" as const, (error: unknown) => error).finally(() => { settled = true; }); + try { + const encrypted = await scratch.started(); + await feed.write(randomBytes(4 * MIB)); + await vi.waitFor(async () => expect((await fs.stat(encrypted)).size).toBeGreaterThanOrEqual(4 * MIB), { timeout: 30_000, interval: 5 }); + for (let request = 0; request < 25; request++) expect(await getWorkspaceSnapshot(db, { ...f, snapshotId: f.row.id })).toBeNull(); + expect(settled).toBe(false); + expect(onThread.bytes()).toBeLessThan(64 * 1024); + // Cancelling stops the encryption at its next chunk and leaves nothing behind. + controller.abort(new Error("run cancelled")); + await feed.write(randomBytes(1024)); + expect(await saving).toMatchObject({ name: "AbortError" }); + expect(await fs.readdir(scratch.scratchRoot)).toEqual([]); + expect(await storedFiles()).toEqual([]); + const [row] = await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)); + expect(row?.status).toBe("pending"); expect(row?.metadata?.uploads ?? []).toEqual([]); + } finally { controller.abort(); await feed.close(); await saving; onThread.restore(); scratch.restore(); } + }, 90_000); + it("digests and decrypts none of a save or a restore on the server thread when the store verified the upload", async () => { + const f = await fixture(randomBytes(8 * MIB)), put = storage.putFile.bind(storage); + storage.putFile = async input => ({ ...await put(input), integrity: "server_verified_md5" }); + const onThread = serverThreadCrypto(); + try { + const ref = await uploadWorkspaceSnapshot(f.upload), restored = path.join(root, "restored-off-thread.pcws"); + await loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath: restored }); + // Wrapping and unwrapping the data key is all that is left here. + expect(onThread.bytes()).toBeLessThan(64 * 1024); + expect((await fs.readFile(restored)).equals(await fs.readFile(f.archivePath))).toBe(true); + } finally { onThread.restore(); } + }); + it("leaves no object behind when a save is cancelled while it is being stored", async () => { + const f = await fixture(randomBytes(MIB)), put = storage.putFile.bind(storage), controller = new AbortController(); + storage.putFile = async input => { controller.abort(new Error("run cancelled")); return put(input); }; + await expect(uploadWorkspaceSnapshot({ ...f.upload, signal: controller.signal })).rejects.toThrow(); + expect(await storedFiles()).toEqual([]); + expect(await getWorkspaceSnapshot(db, { ...f, snapshotId: f.row.id })).toBeNull(); + }); + it.each(["grows", "shrinks", "is rewritten in place"])("rejects an archive that %s after its identity was read, before any upload", async change => { + const f = await fixture(randomBytes(2 * MIB)), put = vi.spyOn(storage, "putFile"); + let reads = 0; + lstatOverride.current = async (file, real) => { + const stats = await real(file); + if (file !== f.archivePath || ++reads !== 1) return stats; + if (change === "grows") appendFileSync(f.archivePath, "one more byte"); + else if (change === "shrinks") truncateSync(f.archivePath, stats.size - 1); + else { const archive = openSync(f.archivePath, "r+"); try { writeSync(archive, randomBytes(64), 0, 64, MIB); } finally { closeSync(archive); } } + return stats; + }; + try { await expect(uploadWorkspaceSnapshot(f.upload)).rejects.toThrow("archive_changed"); } + finally { lstatOverride.current = null; } + expect(put).not.toHaveBeenCalled(); + expect(await storedFiles()).toEqual([]); + }); + it.each([["truncated", "object_hash_mismatch"], ["oversized", "object_size_mismatch"]])("rejects a %s stored object and publishes nothing", async (damage, reason) => { + const f = await fixture(randomBytes(MIB)), ref = await uploadWorkspaceSnapshot(f.upload), objectPath = path.join(root, "objects", ref.objectKey); + if (damage === "truncated") await fs.truncate(objectPath, ref.byteSize - 1); else await fs.appendFile(objectPath, "x"); + const targetPath = path.join(root, `${damage}.pcws`); + await expect(loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath })).rejects.toThrow(reason); + await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await stagedPlaintext()).toEqual([]); + }); + it("rejects a stored object whose ciphertext changed even when the recorded digest was changed to match", async () => { + const f = await fixture(randomBytes(MIB)), ref = await uploadWorkspaceSnapshot(f.upload), objectPath = path.join(root, "objects", ref.objectKey); + const forged = await fs.readFile(objectPath); forged[forged.length - 40]! ^= 1; + await fs.writeFile(objectPath, forged); + const targetPath = path.join(root, "forged.pcws"), scratch = path.join(root, "forged-scratch"); await fs.mkdir(scratch); + await expect(loadWorkspaceSnapshotArchive({ storage, reference: { ...ref, sha256: createHash("sha256").update(forged).digest("hex") }, + targetPath, temporaryParent: scratch })).rejects.toThrow(/authenticate/); + await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await stagedPlaintext()).toEqual([]); expect(await fs.readdir(scratch)).toEqual([]); + }); + it("rejects an archive that decrypts but differs from the digest bound into its key, and removes the staged copy", async () => { + const f = await fixture(randomBytes(MIB)), ref = await uploadWorkspaceSnapshot(f.upload); + for (const change of [{ plaintextSha256: "0".repeat(64) }, { plaintextBytes: f.captured.byteSize + 1 }]) { + const targetPath = path.join(root, `unbound-${Object.keys(change)[0]}.pcws`); + await expect(loadWorkspaceSnapshotArchive({ storage, reference: await rewrap(ref, change), targetPath })).rejects.toThrow("plaintext_hash_mismatch"); + await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await stagedPlaintext()).toEqual([]); + } + }); + it("reports a damaged object before a damaged header, and a damaged header before an unusable key", async () => { + const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload), objectPath = path.join(root, "objects", ref.objectKey); + const shortKey = await rewrap(ref, { dataKey: Buffer.alloc(16, 7).toString("base64") }), targetPath = path.join(root, "ordered.pcws"); + const load = (reference: typeof ref) => loadWorkspaceSnapshotArchive({ storage, reference, targetPath }); + await expect(load(shortKey)).rejects.toThrow("key_invalid"); + const damaged = await fs.readFile(objectPath); damaged[0]! ^= 1; + await fs.writeFile(objectPath, damaged); + await expect(load(shortKey)).rejects.toThrow("object_hash_mismatch"); + await expect(load(ref)).rejects.toThrow("object_hash_mismatch"); + const recorded = createHash("sha256").update(damaged).digest("hex"); + await expect(load({ ...shortKey, sha256: recorded })).rejects.toThrow("ciphertext_header_invalid"); + await expect(load({ ...ref, sha256: recorded })).rejects.toThrow("ciphertext_header_invalid"); + await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await stagedPlaintext()).toEqual([]); + }); it("does not start object retrieval for an already canceled snapshot load", async () => { const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload); const getObject = vi.spyOn(storage, "getObject"); diff --git a/server/src/services/remote-workspace-snapshot-storage.ts b/server/src/services/remote-workspace-snapshot-storage.ts index e20ce211512..16973bf3d69 100644 --- a/server/src/services/remote-workspace-snapshot-storage.ts +++ b/server/src/services/remote-workspace-snapshot-storage.ts @@ -1,4 +1,5 @@ -import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID } from "node:crypto"; +import { createDecipheriv, createHash, randomBytes, randomUUID } from "node:crypto"; +import { decryptFileOffThread, digestFileOffThread, encryptFileOffThread, type FileDecryption, type FileDigest } from "../storage/file-crypto-pool.js"; import { isStorageUnavailableError, sendThroughStorageOutage, storageOutageWaitFor, waitOutStorageOutages } from "../storage/storage-unavailable.js"; import { measureRunPhase, type ExecutionPhase } from "./execution-preparation-timing.js"; import { collectStorageRequestStats, newStorageRequestStats, type StorageRequestStats } from "../storage/request-stats.js"; @@ -57,11 +58,6 @@ function validateReference(ref: WorkspaceSnapshotRef) { || !Number.isSafeInteger(ref.byteSize) || ref.byteSize < MAGIC.length + 12 + 16 || !ref.key || typeof ref.key !== "object") failure("reference_invalid"); } -async function digestFile(file: string, signal?: AbortSignal) { - const digest = createHash("sha256"); let byteSize = 0; - for await (const part of createReadStream(file, { signal, highWaterMark: WORKSPACE_ARCHIVE_IO_BUFFER_BYTES })) { digest.update(part); byteSize += part.length; } - return { sha256: digest.digest("hex"), byteSize }; -} async function verifyStoredSize(storage: StorageService, ref: WorkspaceSnapshotRef, signal?: AbortSignal) { const head = await storage.headObject(ref.companyId, ref.objectKey, { signal }); if (!head.exists || head.contentLength !== ref.byteSize) failure("object_size_mismatch"); @@ -245,17 +241,16 @@ async function uploadWorkspaceSnapshotTimed(input: WorkspaceSnapshotUpload): Pro const file = verified.find(saved => isDeepStrictEqual(saved.ref, ref))!.file; const wrapped = await snapshotKey(ref), handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); try { - const before = await handle.stat(), digest = createHash("sha256"); let bytes = 0; + const before = await handle.stat(); if (!before.isFile() || (before.mode & 0o777) !== 0o600 || before.nlink !== 1 || typeof process.getuid === "function" && before.uid !== process.getuid()) failure("readback_file_changed"); - for await (const chunk of createReadStream(file, { fd: handle.fd, autoClose: false, signal: input.signal, highWaterMark: WORKSPACE_ARCHIVE_IO_BUFFER_BYTES })) { - digest.update(chunk); bytes += chunk.length; - } + // Digested through this open descriptor, off the server's thread. + const kept = await digestFileOffThread({ fd: handle.fd, algorithm: "sha256" }, input.signal); const after = await handle.stat(), named = await lstat(file); if (!named.isFile() || named.isSymbolicLink() || named.dev !== before.dev || named.ino !== before.ino || named.nlink !== 1 || named.mode !== before.mode || after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs - || bytes !== wrapped.plaintextBytes || digest.digest("hex") !== wrapped.plaintextSha256) failure("readback_file_changed"); + || kept.bytes !== wrapped.plaintextBytes || kept.hex !== wrapped.plaintextSha256) failure("readback_file_changed"); await assertParent(); input.signal?.throwIfAborted(); await link(file, target); } finally { await handle.close(); } @@ -321,32 +316,22 @@ async function uploadVerifiedWorkspaceSnapshot(input: WorkspaceSnapshotUpload, v const scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-encrypt-")); try { const encrypted = path.join(scratch, "archive.encrypted"), dataKey = randomBytes(32), iv = randomBytes(12); - const ciphertextDigest = createHash("sha256"), prefix = Buffer.concat([MAGIC, iv]); - let ciphertextBytes = prefix.length; - ciphertextDigest.update(prefix); - const cipher = createCipheriv("aes-256-gcm", dataKey, iv); cipher.setAAD(aad(identity)); - const header = await open(encrypted, "wx", 0o600); - try { await header.writeFile(prefix); } finally { await header.close(); } - // One read of the archive both digests and encrypts it. An unchanged file - // identity across that read replaces the separate digest pass. + // One read of the archive digests it, encrypts it and digests the stored + // bytes, on a worker thread so the server's thread stays free. An unchanged + // file identity across that read replaces the separate digest pass. const before = await lstat(input.archivePath); - const incomingDigest = createHash("sha256"); let incomingBytes = 0; - const observed = new Transform({ transform(bytes, _encoding, callback) { incomingDigest.update(bytes); incomingBytes += bytes.length; callback(null, bytes); } }); - const encryptedDigest = new Transform({ transform(bytes, _encoding, callback) { - ciphertextDigest.update(bytes); ciphertextBytes += bytes.length; callback(null, bytes); - } }); - await timedStep(input.timing, "encryptMs", "storage.snapshot.encrypt", () => pipeline(createReadStream(input.archivePath, { signal: input.signal, highWaterMark: WORKSPACE_ARCHIVE_IO_BUFFER_BYTES }), observed, cipher, encryptedDigest, createWriteStream(encrypted, { flags: "a", mode: 0o600, highWaterMark: WORKSPACE_ARCHIVE_IO_BUFFER_BYTES }), { signal: input.signal })); + const sealed = await timedStep(input.timing, "encryptMs", "storage.snapshot.encrypt", () => encryptFileOffThread({ + source: input.archivePath, target: encrypted, prefix: Buffer.concat([MAGIC, iv]), key: dataKey, iv, aad: aad(identity), + }, input.signal)); const after = await lstat(input.archivePath); - if (!before.isFile() || before.isSymbolicLink() || incomingBytes !== before.size || after.dev !== before.dev || after.ino !== before.ino + if (!before.isFile() || before.isSymbolicLink() || sealed.plaintextBytes !== before.size || after.dev !== before.dev || after.ino !== before.ino || after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs) failure("archive_changed"); - const plaintext = { sha256: incomingDigest.digest("hex"), byteSize: incomingBytes }; - const tag = cipher.getAuthTag(); ciphertextDigest.update(tag); ciphertextBytes += tag.length; - const footer = await open(encrypted, "a"); try { await footer.writeFile(tag); } finally { await footer.close(); } + const plaintext = { sha256: sealed.plaintextSha256, byteSize: sealed.plaintextBytes }; const { material } = await localEncryptedProvider.createVersion({ value: JSON.stringify({ schema: KEY_SCHEMA, ...identity, dataKey: dataKey.toString("base64"), plaintextSha256: plaintext.sha256, plaintextBytes: plaintext.byteSize }) }); dataKey.fill(0); const content = await timedStep(input.timing, "digestMs", "storage.snapshot.digest", async () => ({ - sha256: ciphertextDigest.digest("hex"), byteSize: ciphertextBytes, + sha256: sealed.sha256, byteSize: sealed.byteSize, })); if (input.timing) input.timing.bytes = content.byteSize; const attempt: UploadAttempt = { objectKey: `${identity.companyId}/workspace-snapshots/${identity.workspaceId}/${identity.id}/${randomUUID()}.pcws`, @@ -432,7 +417,7 @@ async function loadWorkspaceSnapshotArchiveTimed(input: Parameters sendThroughStorageOutage(async () => { if (attempts++ > 0) { timer.refresh(); - received = 0; digest = createHash("sha256"); + received = 0; await rm(ciphertext, { force: true }); } const object = await input.storage.getObject(ref.companyId, ref.objectKey, { signal }); const bounded = new Transform({ transform(bytes, _encoding, callback) { if (bytes.length > 0) timer.refresh(); - received += bytes.length; digest.update(bytes); + received += bytes.length; callback(received > ref.byteSize ? new Error("remote_workspace_snapshot_object_size_mismatch") : null, bytes); } }); try { await pipeline(object.stream, bounded, createWriteStream(ciphertext, { flags: "wx", mode: 0o600, highWaterMark: WORKSPACE_ARCHIVE_IO_BUFFER_BYTES }), { signal }); } @@ -461,25 +446,42 @@ async function loadWorkspaceSnapshotArchiveTimed(input: Parameters pipeline(createReadStream(ciphertext, { start: header.length, end: ref.byteSize - tag.length - 1, signal: input.signal, highWaterMark: WORKSPACE_ARCHIVE_IO_BUFFER_BYTES }), - decipher, createWriteStream(plaintext, { flags: "wx", mode: 0o600, highWaterMark: WORKSPACE_ARCHIVE_IO_BUFFER_BYTES }), { signal: input.signal })); + [stored, opened] = await timedStep(input.timing, "decryptMs", "storage.snapshot.decrypt", async () => { + const [digested, decrypted] = await Promise.allSettled([ + digestFileOffThread({ file: ciphertext, algorithm: "sha256" }, input.signal), + key.length === 32 ? decryptFileOffThread({ source: ciphertext, target: plaintext, start: header.length, + end: ref.byteSize - tag.length, key, iv: header.subarray(MAGIC.length), aad: aad(ref), authTag: tag }, input.signal) : undefined, + ]); + if (digested.status === "rejected") throw digested.reason; + if (decrypted.status === "rejected") throw decrypted.reason; + return [digested.value, decrypted.value] as const; + }); } finally { key.fill(0); } - const actual = await timedStep(input.timing, "plaintextVerifyMs", "storage.snapshot.verify_plaintext", () => digestFile(plaintext, input.signal)); - if (actual.sha256 !== wrapped.plaintextSha256 || actual.byteSize !== wrapped.plaintextBytes) failure("plaintext_hash_mismatch"); - // Publish a complete file without replacing an existing destination. A - // failed/cancelled relay cannot leave a plaintext cache that looks complete. + if (stored.bytes !== ref.byteSize || stored.hex !== ref.sha256) failure("object_hash_mismatch"); + if (!header.subarray(0, MAGIC.length).equals(MAGIC)) failure("ciphertext_header_invalid"); + if (!opened) failure("key_invalid"); + if (opened!.authenticationFailure) throw Object.assign(new Error(opened!.authenticationFailure.message), + opened!.authenticationFailure.code ? { code: opened!.authenticationFailure.code } : {}); + // The authenticated archive is digested while it is copied beside the + // destination, so the digest covers exactly the bytes that get published. + // Publication never replaces an existing destination, and a failed or + // cancelled relay cannot leave a plaintext cache that looks complete. const staged = path.join(path.dirname(input.targetPath), `.workspace-snapshot-${randomUUID()}.tmp`); try { + const actual = await timedStep(input.timing, "plaintextVerifyMs", "storage.snapshot.verify_plaintext", () => + digestFileOffThread({ file: plaintext, algorithm: "sha256", copyTo: staged }, input.signal)); + if (actual.hex !== wrapped.plaintextSha256 || actual.bytes !== wrapped.plaintextBytes) failure("plaintext_hash_mismatch"); await timedStep(input.timing, "publishMs", "storage.snapshot.publish", async () => { - await pipeline(createReadStream(plaintext, { signal: input.signal, highWaterMark: WORKSPACE_ARCHIVE_IO_BUFFER_BYTES }), createWriteStream(staged, { flags: "wx", mode: 0o600, highWaterMark: WORKSPACE_ARCHIVE_IO_BUFFER_BYTES }), { signal: input.signal }); input.signal?.throwIfAborted(); await link(staged, input.targetPath); }); } finally { await rm(staged, { force: true }); } diff --git a/server/src/storage/file-crypto-pool.test.ts b/server/src/storage/file-crypto-pool.test.ts new file mode 100644 index 00000000000..fc97ca1f89c --- /dev/null +++ b/server/src/storage/file-crypto-pool.test.ts @@ -0,0 +1,126 @@ +import { createDecipheriv, createHash, randomBytes } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { decryptFileOffThread, digestFileOffThread, encryptFileOffThread, fileCryptoPoolLoad, FILE_CRYPTO_WORKER_LIMIT } from "./file-crypto-pool.js"; + +const MIB = 1024 * 1024; +const sha256 = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex"); + +describe("file crypto pool", () => { + let root: string; + beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-file-crypto-pool-test-")); }); + afterEach(async () => { + expect(fileCryptoPoolLoad()).toEqual({ running: 0, waiting: 0 }); + await fs.rm(root, { recursive: true, force: true }); + }); + const write = async (name: string, bytes: Uint8Array) => { const file = path.join(root, name); await fs.writeFile(file, bytes); return file; }; + /** A file of zeros that takes no disk space and far longer to digest than any test waits. */ + const endless = async (name: string) => { + const file = path.join(root, name), handle = await fs.open(file, "wx", 0o600); + try { await handle.truncate(64 * 1024 * MIB); } finally { await handle.close(); } + return file; + }; + + it("digests a whole file, an inclusive range and an open descriptor as a single-threaded digest would", async () => { + const bytes = randomBytes(3 * MIB + 137), file = await write("source", bytes); + expect(await digestFileOffThread({ file, algorithm: "sha256" })).toEqual({ hex: sha256(bytes), bytes: bytes.length }); + expect(await digestFileOffThread({ file, algorithm: "md5", start: MIB - 5, end: 2 * MIB + 8 })).toEqual({ + hex: createHash("md5").update(bytes.subarray(MIB - 5, 2 * MIB + 9)).digest("hex"), bytes: MIB + 14 }); + // A range that runs past the end of the file stops at the end, as a read stream does. + expect(await digestFileOffThread({ file, algorithm: "sha256", start: bytes.length - 10, end: bytes.length + 99 })).toEqual({ + hex: sha256(bytes.subarray(bytes.length - 10)), bytes: 10 }); + const handle = await fs.open(file, "r"); + try { + expect(await digestFileOffThread({ fd: handle.fd, algorithm: "sha256" })).toEqual({ hex: sha256(bytes), bytes: bytes.length }); + // The descriptor is still the caller's to use and close. + expect((await handle.stat()).size).toBe(bytes.length); + } finally { await handle.close(); } + await expect(digestFileOffThread({ file: path.join(root, "missing"), algorithm: "sha256" })).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("copies exactly the digested bytes to a new owner-only file and never replaces an existing one", async () => { + const bytes = randomBytes(2 * MIB + 3), file = await write("source", bytes), copyTo = path.join(root, "copy"); + expect(await digestFileOffThread({ file, algorithm: "sha256", copyTo })).toEqual({ hex: sha256(bytes), bytes: bytes.length }); + expect((await fs.readFile(copyTo)).equals(bytes)).toBe(true); + expect((await fs.stat(copyTo)).mode & 0o777).toBe(0o600); + await fs.writeFile(copyTo, "kept"); + await expect(digestFileOffThread({ file, algorithm: "sha256", copyTo })).rejects.toMatchObject({ code: "EEXIST" }); + expect(await fs.readFile(copyTo, "utf8")).toBe("kept"); + }); + + it("stores prefix, ciphertext and tag, and reports the digests of the source and of everything stored", async () => { + const bytes = randomBytes(2 * MIB + 77), source = await write("source", bytes), target = path.join(root, "sealed"); + const key = randomBytes(32), kept = Buffer.from(key), iv = randomBytes(12), aad = Buffer.from("bound identity"), prefix = Buffer.from("MAGIC\n"); + const sealed = await encryptFileOffThread({ source, target, prefix, key, iv, aad }); + const stored = await fs.readFile(target); + expect(sealed).toEqual({ plaintextSha256: sha256(bytes), plaintextBytes: bytes.length, sha256: sha256(stored), byteSize: prefix.length + bytes.length + 16 }); + expect((await fs.stat(target)).mode & 0o777).toBe(0o600); + expect(stored.subarray(0, prefix.length).equals(prefix)).toBe(true); + // The caller's key is not consumed: erasing it stays the caller's decision. + expect(key.equals(kept)).toBe(true); + const decipher = createDecipheriv("aes-256-gcm", key, iv); + decipher.setAAD(aad); decipher.setAuthTag(stored.subarray(stored.length - 16)); + expect(Buffer.concat([decipher.update(stored.subarray(prefix.length, stored.length - 16)), decipher.final()]).equals(bytes)).toBe(true); + await expect(encryptFileOffThread({ source, target, prefix, key, iv, aad })).rejects.toMatchObject({ code: "EEXIST" }); + expect((await fs.readFile(target)).equals(stored)).toBe(true); + }); + + it("decrypts an authentic ciphertext and reports every one that does not authenticate", async () => { + const bytes = randomBytes(MIB + 9), source = await write("source", bytes), sealedPath = path.join(root, "sealed"); + const key = randomBytes(32), iv = randomBytes(12), aad = Buffer.from("bound identity"), prefix = Buffer.from("MAGIC\n"); + await encryptFileOffThread({ source, target: sealedPath, prefix, key, iv, aad }); + const stored = await fs.readFile(sealedPath), authTag = stored.subarray(stored.length - 16); + const open = (name: string, change: Partial[0]> = {}) => decryptFileOffThread({ + source: sealedPath, target: path.join(root, name), start: prefix.length, end: stored.length - 16, key, iv, aad, authTag, ...change }); + expect(await open("opened")).toEqual({ authenticationFailure: undefined }); + expect((await fs.readFile(path.join(root, "opened"))).equals(bytes)).toBe(true); + expect((await fs.stat(path.join(root, "opened"))).mode & 0o777).toBe(0o600); + for (const [name, change] of [["other-key", { key: randomBytes(32) }], ["other-aad", { aad: Buffer.from("another identity") }], + ["other-tag", { authTag: randomBytes(16) }]] as const) { + expect(await open(name, change)).toEqual({ authenticationFailure: { message: expect.stringContaining("authenticate") } }); + } + const damaged = Buffer.from(stored); damaged[prefix.length + 5]! ^= 1; + const damagedPath = await write("damaged", damaged); + expect(await open("from-damaged", { source: damagedPath })).toEqual({ authenticationFailure: { message: expect.stringContaining("authenticate") } }); + // A ciphertext cut short does not authenticate either. + expect(await open("from-short", { source: await write("short", stored.subarray(0, stored.length - 40)) })) + .toEqual({ authenticationFailure: { message: expect.stringContaining("authenticate") } }); + }); + + it("runs a bounded number of jobs at once, keeps the rest waiting in order, and cancels each kind promptly", async () => { + const source = await endless("endless"); + const jobs = Array.from({ length: FILE_CRYPTO_WORKER_LIMIT + 2 }, (_, index) => { + // Only the last job, which never runs, is asked to write anything. + const controller = new AbortController(), copyTo = path.join(root, `copy-${index}`); + const outcome = digestFileOffThread({ file: source, algorithm: "md5", ...(index > FILE_CRYPTO_WORKER_LIMIT ? { copyTo } : {}) }, + controller.signal).then(() => "finished", error => error); + return { controller, copyTo, outcome }; + }); + expect(fileCryptoPoolLoad()).toEqual({ running: FILE_CRYPTO_WORKER_LIMIT, waiting: 2 }); + // A waiting job is cancelled at once and never touches its files. + const last = jobs.at(-1)!, reason = new Error("request cancelled"); + last.controller.abort(reason); + expect(await last.outcome).toMatchObject({ name: "AbortError", cause: reason }); + expect(fileCryptoPoolLoad()).toEqual({ running: FILE_CRYPTO_WORKER_LIMIT, waiting: 1 }); + await expect(fs.access(last.copyTo)).rejects.toMatchObject({ code: "ENOENT" }); + // A running job stops between chunks, and its worker takes the next waiting job. + jobs[0]!.controller.abort(reason); + expect(await jobs[0]!.outcome).toMatchObject({ name: "AbortError", cause: reason }); + expect(fileCryptoPoolLoad()).toEqual({ running: FILE_CRYPTO_WORKER_LIMIT, waiting: 0 }); + for (const job of jobs.slice(1, -1)) job.controller.abort(reason); + for (const job of jobs.slice(1, -1)) expect(await job.outcome).toMatchObject({ name: "AbortError" }); + // The workers survive a cancellation and take new work. + const small = await write("small", Buffer.from("after cancellation")); + expect((await digestFileOffThread({ file: small, algorithm: "sha256" })).hex).toBe(sha256(Buffer.from("after cancellation"))); + }, 60_000); + + it("does not start a job whose signal is already aborted", async () => { + const source = await write("source", randomBytes(1024)), target = path.join(root, "sealed"), reason = new Error("already cancelled"); + const controller = new AbortController(); controller.abort(reason); + await expect(encryptFileOffThread({ source, target, prefix: Buffer.alloc(0), key: randomBytes(32), iv: randomBytes(12), aad: Buffer.alloc(0) }, + controller.signal)).rejects.toMatchObject({ name: "AbortError", cause: reason }); + await expect(fs.access(target)).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); diff --git a/server/src/storage/file-crypto-pool.ts b/server/src/storage/file-crypto-pool.ts new file mode 100644 index 00000000000..c8cd2d86f36 --- /dev/null +++ b/server/src/storage/file-crypto-pool.ts @@ -0,0 +1,283 @@ +import { Worker } from "node:worker_threads"; + +/** + * Digests and AES-256-GCM passes over whole files, run on worker threads. + * + * Hashing and ciphering are synchronous in Node: on the server's thread every + * megabyte of a saved workspace delays every request and agent turn behind it. + * A job here names files and byte ranges, never file contents, so a waiting + * job holds no data. At most `FILE_CRYPTO_WORKER_LIMIT` jobs run at once, each + * with two reused read buffers; the rest wait in order and can be cancelled. + */ +export const FILE_CRYPTO_WORKER_LIMIT = 2; + +export interface FileDigestRequest { + /** The file to read, or `fd` for a file the caller already holds open. */ + file?: string; + fd?: number; + /** First and last byte to read, both inclusive. Defaults to the whole file. */ + start?: number; + end?: number; + algorithm: "sha256" | "md5"; + /** Also writes exactly the digested bytes to this new private file. */ + copyTo?: string; +} +export interface FileDigest { hex: string; bytes: number } + +export interface FileEncryptRequest { + source: string; + /** Created here, owner-only: `prefix`, then the ciphertext, then the 16-byte tag. */ + target: string; + prefix: Uint8Array; + key: Uint8Array; + iv: Uint8Array; + aad: Uint8Array; +} +export interface FileEncryption { + plaintextSha256: string; plaintextBytes: number; + /** Digest and size of everything written to `target`. */ + sha256: string; byteSize: number; +} + +export interface FileDecryptRequest { + source: string; + /** Created here, owner-only. Holds unauthenticated bytes until the result says otherwise. */ + target: string; + /** First ciphertext byte, and the byte after the last one. */ + start: number; + end: number; + key: Uint8Array; + iv: Uint8Array; + aad: Uint8Array; + authTag: Uint8Array; +} +export interface FileDecryption { + /** Set when the ciphertext, key, additional data or tag did not authenticate. */ + authenticationFailure?: { message: string; code?: string }; +} + +type Job = ({ kind: "digest" } & FileDigestRequest) | ({ kind: "encrypt" } & FileEncryptRequest) | ({ kind: "decrypt" } & FileDecryptRequest); +type WorkerRequest = { id: number; job: Job; cancel: Int32Array }; +type WorkerReply = { id: number; result?: unknown; cancelled?: true; + error?: { name?: string; message: string; code?: string; errno?: number; syscall?: string; path?: string } }; + +/** + * The worker thread's whole program. It is sent as source text, so it may use + * only its `require` argument and JavaScript built-ins, never this module's + * bindings. One job runs at a time, through two reused read buffers. + */ +function fileCryptoWorker(require: NodeJS.Require): void { + const { parentPort } = require("node:worker_threads") as typeof import("node:worker_threads"); + const fs = require("node:fs") as typeof import("node:fs"); + const crypto = require("node:crypto") as typeof import("node:crypto"); + const { promisify } = require("node:util") as typeof import("node:util"); + const readFrom = promisify(fs.read), writeTo = promisify(fs.write); + const chunks = [Buffer.allocUnsafe(1024 * 1024), Buffer.allocUnsafe(1024 * 1024)]; + const cancelled = {}; + const bytes = (value: Uint8Array) => Buffer.from(value.buffer, value.byteOffset, value.byteLength); + const writeAll = async (fd: number, data: Uint8Array) => { + for (let offset = 0; offset < data.length;) offset += (await writeTo(fd, data, offset, data.length - offset, null)).bytesWritten; + }; + /** + * Reads `[start, end)` of `input` in order, up to the end of the file, and + * writes what `each` returns for every chunk to `output`, in the same order. + * Without a `start` it reads on from the descriptor's own position, as the + * stream it replaces did. One write is in flight while the next chunk is + * read and processed, so the disk and this thread work at the same time. + */ + const pump = async (input: number, start: number | undefined, end: number, cancel: Int32Array, output: number | undefined, + each: (part: Buffer) => Uint8Array) => { + const first = start ?? 0; + let position = first, turn = 0, writing = Promise.resolve(), written: ArrayBufferLike | undefined; + try { + while (position < end) { + if (Atomics.load(cancel, 0) !== 0) throw cancelled; + const chunk = chunks[turn]!; turn ^= 1; + // A copy writes the read buffer itself: never read into one still being written. + if (written === chunk.buffer) await writing; + const { bytesRead } = await readFrom(input, chunk, 0, Math.min(chunk.length, end - position), start === undefined ? null : position); + if (bytesRead === 0) break; + const data = each(chunk.subarray(0, bytesRead)); + position += bytesRead; + if (output === undefined || data.length === 0) continue; + await writing; + writing = writeAll(output, data); written = data.buffer; + // A failure is reported by the next `await writing`, not as an unhandled rejection. + writing.catch(() => undefined); + } + await writing; + } catch (error) { + // No file is closed while a write to it is still in flight. + await writing.catch(() => undefined); + throw error; + } + return position - first; + }; + const close = (fd: number | undefined) => { if (fd !== undefined) fs.closeSync(fd); }; + const run = async (job: Job, cancel: Int32Array): Promise => { + let input: number | undefined, output: number | undefined; + try { + if (job.kind === "digest") { + input = job.fd ?? fs.openSync(job.file!, "r"); + if (job.copyTo !== undefined) output = fs.openSync(job.copyTo, "wx", 0o600); + const digest = crypto.createHash(job.algorithm); + const count = await pump(input, job.start, job.end === undefined ? Infinity : job.end + 1, cancel, output, part => { + digest.update(part); + return part; + }); + return { hex: digest.digest("hex"), bytes: count }; + } + const key = bytes(job.key); + try { + input = fs.openSync(job.source, "r"); + const target = output = fs.openSync(job.target, "wx", 0o600); + if (job.kind === "encrypt") { + const plaintext = crypto.createHash("sha256"), stored = crypto.createHash("sha256"); + const cipher = crypto.createCipheriv("aes-256-gcm", key, bytes(job.iv)); + cipher.setAAD(bytes(job.aad)); + let byteSize = 0; + const store = (data: Uint8Array) => { stored.update(data); byteSize += data.length; return data; }; + await writeAll(target, store(bytes(job.prefix))); + const plaintextBytes = await pump(input, undefined, Infinity, cancel, target, part => { plaintext.update(part); return store(cipher.update(part)); }); + await writeAll(target, store(cipher.final())); + await writeAll(target, store(cipher.getAuthTag())); + return { plaintextSha256: plaintext.digest("hex"), plaintextBytes, sha256: stored.digest("hex"), byteSize }; + } + const decipher = crypto.createDecipheriv("aes-256-gcm", key, bytes(job.iv)); + decipher.setAAD(bytes(job.aad)); decipher.setAuthTag(bytes(job.authTag)); + await pump(input, job.start, job.end, cancel, target, part => decipher.update(part)); + let last: Buffer | undefined, authenticationFailure: { message: string; code?: string } | undefined; + try { last = decipher.final(); } + catch (error) { authenticationFailure = { message: (error as Error).message, code: (error as NodeJS.ErrnoException).code }; } + if (last) await writeAll(target, last); + return { authenticationFailure }; + } finally { key.fill(0); } + } finally { + // A caller's own descriptor stays open for the caller to close. + try { close(output); } finally { if (job.kind !== "digest" || job.fd === undefined) close(input); } + } + }; + parentPort!.on("message", (request: WorkerRequest) => { + run(request.job, request.cancel).then((result): WorkerReply => ({ id: request.id, result }), (error): WorkerReply => { + const failure = error as NodeJS.ErrnoException; + return error === cancelled ? { id: request.id, cancelled: true } : { id: request.id, error: { name: failure?.name, + message: String(failure?.message ?? error), code: failure?.code, errno: failure?.errno, syscall: failure?.syscall, path: failure?.path } }; + }).then(reply => parentPort!.postMessage(reply)); + }); +} + +// `__name` is a helper some TypeScript loaders wrap around nested functions. +const WORKER_SOURCE = `const __name = value => value;\n(${fileCryptoWorker.toString()})(require);`; + +type Pending = { + id: number; job: Job; cancel: Int32Array; signal?: AbortSignal; onAbort?: () => void; + resolve: (result: unknown) => void; reject: (error: unknown) => void; +}; +const queue: Pending[] = []; +const idle: Worker[] = []; +const running = new Map(); +let nextId = 0; + +/** What a cancelled stream pipeline rejects with, so callers see the same failure as before. */ +function abortError(signal: AbortSignal) { + return Object.assign(new Error("The operation was aborted", { cause: signal.reason }), { name: "AbortError", code: "ABORT_ERR" }); +} + +const secret = (job: Job) => (job.kind === "digest" ? undefined : job.key); + +function settle(pending: Pending) { + if (pending.onAbort) pending.signal!.removeEventListener("abort", pending.onAbort); + // A key that never reached a worker is still this thread's copy to erase. + const key = secret(pending.job); + if (key && key.byteLength > 0) key.fill(0); +} + +function release(worker: Worker, healthy: boolean) { + running.delete(worker); + if (healthy) { worker.unref(); idle.push(worker); } + else void worker.terminate(); + dispatch(); +} + +function spawn() { + const worker = new Worker(WORKER_SOURCE, { eval: true, execArgv: [], + resourceLimits: { maxOldGenerationSizeMb: 64, maxYoungGenerationSizeMb: 16, stackSizeMb: 4 } }); + const fail = (error: Error) => { + const index = idle.indexOf(worker); + if (index >= 0) idle.splice(index, 1); + const pending = running.get(worker); + if (!pending && index < 0) return; + if (pending) { settle(pending); pending.reject(error); } + release(worker, false); + }; + worker.on("message", (reply: WorkerReply) => { + const pending = running.get(worker); + if (!pending || pending.id !== reply.id) return; + settle(pending); + if (reply.cancelled) pending.reject(abortError(pending.signal!)); + else if (reply.error) pending.reject(Object.assign(new Error(reply.error.message), reply.error)); + else pending.resolve(reply.result); + release(worker, true); + }); + worker.on("error", fail); + worker.on("exit", () => fail(new Error("file_crypto_worker_exited"))); + return worker; +} + +function dispatch() { + while (queue.length && (idle.length || running.size < FILE_CRYPTO_WORKER_LIMIT)) { + const pending = queue.shift()!; + let worker: Worker; + try { worker = idle.pop() ?? spawn(); } + catch (error) { settle(pending); pending.reject(error); continue; } + worker.ref(); + running.set(worker, pending); + const key = secret(pending.job); + try { worker.postMessage({ id: pending.id, job: pending.job, cancel: pending.cancel } satisfies WorkerRequest, key ? [key.buffer as ArrayBuffer] : []); } + catch (error) { settle(pending); pending.reject(error); release(worker, false); } + } +} + +function submit(job: Job, signal: AbortSignal | undefined): Promise { + return new Promise((resolve, reject) => { + const pending: Pending = { id: nextId++, job, cancel: new Int32Array(new SharedArrayBuffer(4)), signal, + resolve: resolve as (result: unknown) => void, reject }; + if (signal?.aborted) { settle(pending); reject(abortError(signal)); return; } + if (signal) { + pending.onAbort = () => { + const index = queue.indexOf(pending); + // A running job stops at its next chunk and answers once its files are closed. + if (index < 0) { Atomics.store(pending.cancel, 0, 1); return; } + queue.splice(index, 1); settle(pending); reject(abortError(signal)); + }; + signal.addEventListener("abort", pending.onAbort, { once: true }); + } + queue.push(pending); + dispatch(); + }); +} + +/** Jobs running and waiting right now. */ +export function fileCryptoPoolLoad() { + return { running: running.size, waiting: queue.length }; +} + +export function digestFileOffThread(request: FileDigestRequest, signal?: AbortSignal): Promise { + return submit({ kind: "digest", ...request }, signal); +} + +// Each byte array is copied to its exact size, so nothing else that shares its +// allocation crosses to the worker. The key's copy is moved, not cloned. +const exact = (value: Uint8Array) => new Uint8Array(value); + +/** The worker erases its copy of `key`; the caller's stays the caller's to erase. */ +export function encryptFileOffThread(request: FileEncryptRequest, signal?: AbortSignal): Promise { + return submit({ kind: "encrypt", ...request, prefix: exact(request.prefix), key: exact(request.key), iv: exact(request.iv), + aad: exact(request.aad) }, signal); +} + +/** The worker erases its copy of `key`; the caller's stays the caller's to erase. */ +export function decryptFileOffThread(request: FileDecryptRequest, signal?: AbortSignal): Promise { + return submit({ kind: "decrypt", ...request, key: exact(request.key), iv: exact(request.iv), aad: exact(request.aad), + authTag: exact(request.authTag) }, signal); +} diff --git a/server/src/storage/s3-multipart.ts b/server/src/storage/s3-multipart.ts index 515faa56d45..d782d360fb0 100644 --- a/server/src/storage/s3-multipart.ts +++ b/server/src/storage/s3-multipart.ts @@ -1,7 +1,6 @@ import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, UploadPartCommand, type S3Client } from "@aws-sdk/client-s3"; import { sendThroughStorageOutage } from "./storage-unavailable.js"; import { recordStorageBackoff, recordStoragePart, recordStorageThrottle, withStorageRequestStats } from "./request-stats.js"; -import { createHash } from "node:crypto"; import { once } from "node:events"; import { createReadStream } from "node:fs"; import { stat } from "node:fs/promises"; @@ -10,6 +9,7 @@ import { setTimeout as delay } from "node:timers/promises"; import type { PutObjectInput, PutObjectReceipt } from "./types.js"; import { withS3FailureDiagnostics } from "./s3-failure.js"; import { cleanupStorageFailure, STORAGE_FAILURE_CLEANUP_MS } from "./failure-cleanup.js"; +import { digestFileOffThread } from "./file-crypto-pool.js"; const bufferedPartSize = 8 * 1024 * 1024; const bufferedPayloadLimit = 4 * bufferedPartSize; @@ -160,7 +160,6 @@ const FILE_PART_ATTEMPTS = 3; export const THROTTLED_PART_ATTEMPTS = 8; const THROTTLE_BACKOFF_BASE_MS = 1_000; const THROTTLE_BACKOFF_MAX_MS = 20_000; -const FILE_READ_BYTES = 1024 * 1024; /** * S3 answers a request-rate limit with 503 SlowDown; compatible gateways also @@ -210,11 +209,10 @@ export async function sendFileRequestWithRetry(signal: AbortSignal | undefine } } -/** Base64 MD5 of an inclusive byte range, as the Content-MD5 header expects. */ +/** Base64 MD5 of an inclusive byte range, as the Content-MD5 header expects. Hashed off the server's thread. */ export async function fileRangeMd5(file: string, start: number, end: number, signal?: AbortSignal) { - const digest = createHash("md5"); - for await (const chunk of createReadStream(file, { start, end, signal, highWaterMark: FILE_READ_BYTES })) digest.update(chunk); - return digest.digest("base64"); + const digest = await digestFileOffThread({ file, start, end, algorithm: "md5" }, signal); + return Buffer.from(digest.hex, "hex").toString("base64"); } export async function assertSourceFileSize(file: string, contentLength: number) { @@ -253,9 +251,18 @@ export async function putS3MultipartFile( // only for requests sent after the previous one, so a burst of parallel // throttled parts halves the rate once rather than once per part. let concurrency = FILE_PART_CONCURRENCY, reductions = 0; + const partMd5 = (index: number) => fileRangeMd5(file, index * partSize, Math.min((index + 1) * partSize, input.contentLength) - 1, controller.signal); + // The first round of parts is digested before any of them is sent. They + // leave together, as they did when this thread digested them side by side, + // so a rate limit still answers one burst rather than a trickle of requests. + // Every digest of the round has ended, also a failed or cancelled one, + // before a part goes on: none still reads the file when the upload settles. + const firstRound = Array.from({ length: Math.min(FILE_PART_CONCURRENCY, count) }, (_, index) => partMd5(index)); + const firstRoundEnded = Promise.allSettled(firstRound); const sendPart = async (index: number) => { const start = index * partSize, length = Math.min(partSize, input.contentLength - start), end = start + length - 1; - const contentMd5 = await fileRangeMd5(file, start, end, controller.signal); + if (index < firstRound.length) await firstRoundEnded; + const contentMd5 = await (firstRound[index] ?? partMd5(index)); for (let failures = 0, throttles = 0; ;) { controller.signal.throwIfAborted(); const body = createReadStream(file, { start, end, highWaterMark: 64 * 1024 });