diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 2da5478d001..964850fdbe0 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -723,7 +723,7 @@ run phases: | `workspace.snapshot.sandbox_upload` | Inside `sandbox_restore`: copy the archive into the sandbox. | | `workspace.snapshot.sandbox_extract` | Inside `sandbox_restore`: verify and unpack the archive, then publish the restored folder. | | `storage.snapshot.prior_attempts` | Check whether an earlier upload attempt already landed. | -| `storage.snapshot.encrypt` | Hash the archive, encrypt it and hash the encrypted file, in one pass on a worker thread. | +| `storage.snapshot.encrypt` | Hash the archive, encrypt it and hash the encrypted file, in one pass on a worker thread. This is the only full read of a downloaded archive: one whose digest or size differs from the sandbox's receipt fails here, before any upload is recorded. | | `storage.snapshot.digest` | Record the digest of the encrypted file. The `encrypt` pass computes it, so this step takes no time. | | `storage.snapshot.put` | Upload the encrypted archive (multipart above 16 MiB). | | `storage.snapshot.verify` | Confirm the stored object (its size when the backend checked every part, otherwise a full read-back). | diff --git a/server/src/services/remote-workspace-archive-io.ts b/server/src/services/remote-workspace-archive-io.ts index af338322d85..bc21e43be6d 100644 --- a/server/src/services/remote-workspace-archive-io.ts +++ b/server/src/services/remote-workspace-archive-io.ts @@ -1 +1,2 @@ export const WORKSPACE_ARCHIVE_IO_BUFFER_BYTES = 1024 * 1024; +export const REMOTE_WORKSPACE_CAPTURE_TRANSFER_MISMATCH = "remote_workspace_capture_transfer_mismatch"; diff --git a/server/src/services/remote-workspace-snapshot-storage.test.ts b/server/src/services/remote-workspace-snapshot-storage.test.ts index b2aa609e849..6749677c0a7 100644 --- a/server/src/services/remote-workspace-snapshot-storage.test.ts +++ b/server/src/services/remote-workspace-snapshot-storage.test.ts @@ -234,6 +234,18 @@ suite("remote workspace snapshot storage", () => { expect(reads).toBe(2); expect(put).not.toHaveBeenCalled(); }); + it.each([["digest", 0], ["size", 1]] as const)("rejects an archive whose %s differs from the sandbox receipt before recording or storing it", async (_field, extra) => { + const f = await fixture(); const put = vi.spyOn(storage, "putFile"); + const expectedArchive = { sha256: extra ? f.captured.sha256 : createHash("sha256").update("other archive").digest("hex"), + byteSize: f.captured.byteSize + extra }; + await expect(uploadWorkspaceSnapshot({ ...f.upload, expectedArchive })).rejects.toThrow("remote_workspace_capture_transfer_mismatch"); + expect(put).not.toHaveBeenCalled(); + const [row] = await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)); + expect(row).toMatchObject({ status: "pending", objectKey: null, sha256: null }); + expect(row?.metadata?.uploads ?? []).toEqual([]); + const ref = await uploadWorkspaceSnapshot({ ...f.upload, expectedArchive: { sha256: f.captured.sha256, byteSize: f.captured.byteSize } }); + expect(ref.manifestDigest).toBe(f.captured.manifestDigest); + }); it("reconciles an upload whose reply was lost after object persistence, without the old scratch file", async () => { const f = await fixture(); const put = storage.putFile.bind(storage); let loseReply = true; storage.putFile = async input => { const result = await put(input); if (loseReply) { loseReply = false; throw new Error("lost upload reply"); } return result; }; diff --git a/server/src/services/remote-workspace-snapshot-storage.ts b/server/src/services/remote-workspace-snapshot-storage.ts index 16973bf3d69..a6abb807021 100644 --- a/server/src/services/remote-workspace-snapshot-storage.ts +++ b/server/src/services/remote-workspace-snapshot-storage.ts @@ -16,7 +16,7 @@ import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js"; import type { StoredSecretVersionMaterial } from "../secrets/types.js"; import type { StorageService } from "../storage/types.js"; import { abortSnapshotAttemptUploads, notifyRemoteWorkspaceSnapshots, snapshotAttemptKeys, REMOTE_WORKSPACE_SNAPSHOT_GRACE_MS } from "./remote-workspace-snapshot-retention.js"; -import { WORKSPACE_ARCHIVE_IO_BUFFER_BYTES } from "./remote-workspace-archive-io.js"; +import { REMOTE_WORKSPACE_CAPTURE_TRANSFER_MISMATCH, WORKSPACE_ARCHIVE_IO_BUFFER_BYTES } from "./remote-workspace-archive-io.js"; const SCHEMA = "paperclip.workspace-snapshot.v1"; const KEY_SCHEMA = "paperclip.workspace-snapshot-key.v1"; @@ -199,6 +199,7 @@ type WorkspaceSnapshotUpload = { /** Receives per-step durations and storage request counts. */ timing?: WorkspaceSnapshotStorageTiming; verification?: "readback" | "upload"; + expectedArchive?: { sha256: string; byteSize: number }; }; /** Upload attempts are journaled before remote I/O. The caller owns publication. */ @@ -326,6 +327,8 @@ async function uploadVerifiedWorkspaceSnapshot(input: WorkspaceSnapshotUpload, v const after = await lstat(input.archivePath); if (!before.isFile() || before.isSymbolicLink() || sealed.plaintextBytes !== before.size || after.dev !== before.dev || after.ino !== before.ino || after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs) failure("archive_changed"); + if (input.expectedArchive && (sealed.plaintextSha256 !== input.expectedArchive.sha256 + || sealed.plaintextBytes !== input.expectedArchive.byteSize)) throw new Error(REMOTE_WORKSPACE_CAPTURE_TRANSFER_MISMATCH); const plaintext = { sha256: sealed.plaintextSha256, byteSize: sealed.plaintextBytes }; const { material } = await localEncryptedProvider.createVersion({ value: JSON.stringify({ schema: KEY_SCHEMA, ...identity, dataKey: dataKey.toString("base64"), plaintextSha256: plaintext.sha256, plaintextBytes: plaintext.byteSize }) }); diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index 8197db8ec2c..f49487d126e 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -26,6 +26,16 @@ import { createRunPreparationTiming, drainRunPreparationPhases, measureRunPhase import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; import { readWorkspaceInventory, readonlyWorkspaceInventoryScript } from "./remote-workspace-readonly-inventory.js"; +const archiveReads = vi.hoisted(() => ({ paths: [] as string[] })); +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + const createReadStream = ((file: Parameters[0], ...rest: unknown[]) => { + archiveReads.paths.push(String(file)); + return (actual.createReadStream as (...args: unknown[]) => ReturnType)(file, ...rest); + }) as typeof actual.createReadStream; + return { ...actual, createReadStream, default: { ...actual, createReadStream } }; +}); + const exec = promisify(execFile); describe("remote workspace transport safety with real sandbox commands", () => { @@ -243,6 +253,28 @@ describe("remote workspace transport safety with real sandbox commands", () => { } finally { await captured.cleanup(); } }); + it("leaves the downloaded archive's single full read to the encrypting upload", async () => { + await fs.writeFile(path.join(workspace, "large.bin"), randomBytes(2 * 1024 * 1024 + 11)); + archiveReads.paths.length = 0; + const captured = await captureRemoteWorkspaceArchive({ target: target().remote, remoteCwd: workspace, operationId: randomUUID() }); + try { + expect(archiveReads.paths.filter(file => file === captured.archivePath)).toEqual([]); + expect((await fs.stat(captured.archivePath)).size).toBe(captured.byteSize); + } finally { await captured.cleanup(); } + }); + + it("rejects a downloaded archive whose size differs from the sandbox receipt", async () => { + await fs.writeFile(path.join(workspace, "saved.txt"), "saved work\n"); + const remote = target().remote, syncOut = remote.runner!.syncOut!; + remote.runner!.syncOut = async (operations, options) => { + const result = await syncOut(operations, options); + for (const operation of operations) for (const file of operation.files) await fs.appendFile(file.targetPath, "x"); + return result; + }; + await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) + .rejects.toThrow("remote_workspace_capture_transfer_mismatch"); + }); + it("times the restore's sandbox upload and extraction as separate run-log steps", async () => { await fs.writeFile(path.join(workspace, "saved.txt"), "saved synthetic files\n"); const { remote, execute } = target(); diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 79e2fac753f..8e80375eecb 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -1,6 +1,5 @@ -import { createHash, randomUUID } from "node:crypto"; -import { createReadStream } from "node:fs"; -import { mkdtemp, realpath, rm, stat } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { lstat, mkdtemp, realpath, rm, stat } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; @@ -12,7 +11,7 @@ import { measureRunPhase } from "./execution-preparation-timing.js"; import type { GitRemoteAuthProvider } from "./git-credentials.js"; import { classifyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-merge"; import { MAX_SANDBOX_SYNC_TIMEOUT_MS, type SandboxSyncOperation } from "@paperclipai/adapter-utils/sandbox-managed-runtime"; -import { WORKSPACE_ARCHIVE_IO_BUFFER_BYTES } from "./remote-workspace-archive-io.js"; +import { REMOTE_WORKSPACE_CAPTURE_TRANSFER_MISMATCH } from "./remote-workspace-archive-io.js"; export type RemoteWorkspaceTarget = Extract; @@ -474,8 +473,6 @@ export type RemoteWorkspaceSavePhaseTiming = { captureMs?: number; /** Provider download of the captured archive. */ transferMs?: number; - /** Server-side re-hash of the downloaded archive. */ - verifyMs?: number; byteSize?: number; entries?: number; }; @@ -626,15 +623,12 @@ async function captureSandboxWorkspaceArchive(input: { files: [{ kind: "file", sourcePath: layout === "captured" ? path.posix.join(staged.directory, "captured", "workspace.pcws") : staged.archive, targetPath: archivePath }], }], receipt.byteSize); timing.transferMs = elapsedSince(transferStarted); - const verifyStarted = performance.now(); - const hash = createHash("sha256"); let bytes = 0; - for await (const chunk of createReadStream(archivePath, { highWaterMark: WORKSPACE_ARCHIVE_IO_BUFFER_BYTES })) { hash.update(chunk); bytes += chunk.length; } - timing.verifyMs = elapsedSince(verifyStarted); - if (hash.digest("hex") !== receipt.sha256 || bytes !== receipt.byteSize) throw new Error("remote_workspace_capture_transfer_mismatch"); + const transferred = await lstat(archivePath); + if (!transferred.isFile() || transferred.size !== receipt.byteSize) throw new Error(REMOTE_WORKSPACE_CAPTURE_TRANSFER_MISMATCH); return { ...receipt, archivePath, cleanup: () => rm(root, { recursive: true, force: true }) }; } catch (error) { await rm(root, { recursive: true, force: true }); - if (!(error instanceof Error && error.message === "remote_workspace_capture_transfer_mismatch")) noteProviderRefusal("download", error); + if (!(error instanceof Error && error.message === REMOTE_WORKSPACE_CAPTURE_TRANSFER_MISMATCH)) noteProviderRefusal("download", error); throw error; } } diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index 88cb8cf0a8c..c3d3d8f29dd 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -1262,6 +1262,38 @@ require('node:module').syncBuiltinESMExports();`); expect(operations.at(-1)).toMatchObject({ status: "succeeded" }); }, 60_000); + it("rejects a transfer that changed the archive's bytes without storing it, and saves on retry", async () => { + const reserved = await reserve({ kind: "empty" }), remote = await target("corrupted-transfer"); + const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: remote }); + await fs.writeFile(path.join(prepared.remoteRoot, "saved.txt"), "work that must be saved intact\n"); + const syncOut = remote.runner!.syncOut!; + remote.runner!.syncOut = async (operations, options) => { + const result = await syncOut(operations, options); + for (const operation of operations) for (const file of operation.files) { + const bytes = await fs.readFile(file.targetPath); bytes[bytes.length - 1] ^= 0xff; + await fs.writeFile(file.targetPath, bytes); + } + return result; + }; + const put = vi.spyOn(storage, "putFile"); + await expect(saveRemoteWorkspace({ db, storage, workspace: prepared.workspace, runId: ids.run, lease: prepared.lease, + target: remote, boundary: "final", assertOwned: async () => undefined })).rejects.toThrow("remote_workspace_capture_transfer_mismatch"); + expect(put).not.toHaveBeenCalled(); + const pending = await db.select().from(executionWorkspaceSnapshots).where(and( + eq(executionWorkspaceSnapshots.executionWorkspaceId, reserved.id), eq(executionWorkspaceSnapshots.status, "pending"))); + expect(pending.map(row => row.metadata?.uploads ?? [])).toEqual([[]]); + expect(readRemoteWorkspaceAuthority((await workspace(reserved.id)).metadata)).toMatchObject({ dirty: true, writerRunId: ids.run, + snapshotId: readRemoteWorkspaceAuthority(prepared.workspace.metadata)!.snapshotId }); + remote.runner!.syncOut = syncOut; + const saved = await saveRemoteWorkspace({ db, storage, workspace: await workspace(reserved.id), runId: ids.run, lease: prepared.lease, + target: remote, boundary: "final", assertOwned: async () => undefined }); + expect(put).toHaveBeenCalledTimes(1); + await fs.rm(remote.remoteCwd, { recursive: true, force: true }); + const restored = await createSavedWorkspaceLoader(db, storage)({ companyId: ids.company, workspaceId: reserved.id, snapshotId: saved.snapshotId }); + try { expect(await fs.readFile(path.join(restored.root, "saved.txt"), "utf8")).toBe("work that must be saved intact\n"); } + finally { await restored.cleanup(); } + }, 60_000); + it("retains the writer and previous snapshot when object storage fails", async () => { const reserved = await reserve({ kind: "empty" }), first = await target("outage-sandbox"); const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first }); @@ -1279,7 +1311,7 @@ require('node:module').syncBuiltinESMExports();`); sql`${workspaceOperations.metadata}->>'boundary' = 'final'`))).map(save => save.metadata?.saveTiming); const duration = expect.any(Number), archive = { byteSize: expect.any(Number), entries: expect.any(Number) }; expect(await finalTiming()).toEqual([{ schema: "paperclip.remote-workspace-save-timing.v1", outcome: "failed", totalMs: duration, - captureMs: duration, transferMs: duration, verifyMs: duration, ...archive, storage: expect.objectContaining({ requests: expect.any(Object) }), + captureMs: duration, transferMs: duration, ...archive, storage: expect.objectContaining({ requests: expect.any(Object) }), at: expect.any(String) }]); const beforeSuccess = await workspace(reserved.id); const saved = await saveRemoteWorkspace({ db, storage, workspace: beforeSuccess, runId: ids.run, @@ -1291,7 +1323,7 @@ require('node:module').syncBuiltinESMExports();`); lease: prepared.lease, target: first, assertOwned: async () => undefined })).resolves.toMatchObject({ snapshotId: saved.snapshotId }); expect(await finalTiming()).toEqual([succeededTiming]); expect(await finalTiming()).toEqual([{ schema: "paperclip.remote-workspace-save-timing.v1", outcome: "succeeded", totalMs: duration, - captureMs: duration, transferMs: duration, verifyMs: duration, uploadMs: duration, publishMs: duration, ...archive, + captureMs: duration, transferMs: duration, uploadMs: duration, publishMs: duration, ...archive, storage: expect.objectContaining({ encryptMs: duration, digestMs: duration, putMs: duration, verifyMs: duration, bytes: expect.any(Number), requests: expect.any(Object) }), at: expect.any(String) }]); diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index 5e2ebec34a7..14b8939aada 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -926,7 +926,8 @@ export async function saveRemoteWorkspace(input: { archiveSha256: captured.sha256, archiveBytes: captured.byteSize, timing: storageTiming }) : null; const saved = unchanged ?? await uploadWorkspaceSnapshot({ db: input.db, storage, snapshotId: snapshot.id, companyId: input.workspace.companyId, workspaceId: input.workspace.id, archivePath: captured?.archivePath ?? "", - manifestDigest: existingUpload?.manifestDigest ?? captured!.manifestDigest, timing: storageTiming }); + manifestDigest: existingUpload?.manifestDigest ?? captured!.manifestDigest, timing: storageTiming, + ...(captured ? { expectedArchive: { sha256: captured.sha256, byteSize: captured.byteSize } } : {}) }); timing.uploadMs = Math.max(0, Math.round(performance.now() - uploadStarted)); await input.assertOwned(); const publishStarted = performance.now();