diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 0cea0e95403..ae2d25e3485 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1075,7 +1075,13 @@ transfer's own scratch paths and the explicit source exclusions. Older providers that omit files fail inventory verification rather than publish an incomplete save. Fallback inventory is limited to 100,000 entries, 64 standalone repositories, 1 MiB of configuration, 1,024 settings and 256 refspec/merge entries. Controller -verification and archive normalization share a two-minute cancellation deadline. +verification shares a two-minute budget across the received-tree check and both +source rechecks. Controller archive normalization has its own five-minute +cancellation deadline, matching normal sandbox capture. Time spent sealing does +not consume the remaining metadata verification budget. The fallback does not +extend a caller deadline or create new execution authority. Its save interface +currently has no caller abort signal; the existing source ownership checks still +fence durable publication. These limits reject excessive work before publication and keep the source retained. Inventory rechecks cancellation after file identity validation and before opening the read stream. An expired deadline rejects the capture without an unhandled diff --git a/server/src/services/remote-workspace-readonly-capture-budget.test.ts b/server/src/services/remote-workspace-readonly-capture-budget.test.ts new file mode 100644 index 00000000000..ea184da6c51 --- /dev/null +++ b/server/src/services/remote-workspace-readonly-capture-budget.test.ts @@ -0,0 +1,166 @@ +import { execFile } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import * as archiveOwner from "./remote-workspace-snapshot-archive.js"; +import { readWorkspaceInventory } from "./remote-workspace-readonly-inventory.js"; +import { captureRemoteWorkspaceArchive, REMOTE_WORKSPACE_SANDBOX_DISK_FULL, type RemoteWorkspaceTarget } from "./remote-workspace-transport.js"; + +const exec = promisify(execFile); +let root: string; +let source: string; +async function git(cwd: string, ...args: string[]) { + return exec("git", ["-C", cwd, "-c", "user.name=Budget fixture", "-c", "user.email=fixture@example.invalid", + "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", ...args], { + maxBuffer: 32 * 1024 ** 2, + env: { PATH: process.env.PATH, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1", GIT_TERMINAL_PROMPT: "0" }, + }); +} + +beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "readonly-capture-budget-"))); + source = path.join(root, "source"); + await fs.mkdir(source); + await git(source, "init", "--initial-branch=main"); + for (let directory = 0; directory < 10; directory++) { + const current = path.join(source, `part-${directory}`); + await fs.mkdir(current); + for (let file = 0; file < 40; file++) await fs.writeFile(path.join(current, `item-${file}.txt`), Buffer.alloc(4096, file)); + } + await fs.writeFile(path.join(source, "result.bin"), Buffer.alloc(8 * 1024 ** 2, 97)); + await git(source, "add", "."); + await git(source, "commit", "-m", "Synthetic source"); + await fs.writeFile(path.join(source, "result.bin"), Buffer.alloc(8 * 1024 ** 2, 98)); + await git(source, "add", "result.bin"); + await fs.writeFile(path.join(source, "untracked.txt"), "final dirty workspace bytes\n"); + await fs.symlink("untracked.txt", path.join(source, "result-link")); +}); + +afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); +}); + +it("seals and restores the actual standalone source within a fresh existing verification budget", async () => { + const before = await readWorkspaceInventory(source); + const archivePath = path.join(root, "control.pcws"); + const saved = await archiveOwner.captureRemoteWorkspaceDirectory({ cwd: source, archivePath, + exclude: before.exclude, signal: AbortSignal.timeout(120_000) }); + const restored = path.join(root, "restored"); + await archiveOwner.restoreRemoteWorkspaceArchive({ archivePath, targetDir: restored, expectedManifestDigest: saved.manifestDigest }); + expect((await git(restored, "rev-parse", "HEAD")).stdout).toBe((await git(source, "rev-parse", "HEAD")).stdout); + expect(createHash("sha256").update(await fs.readFile(path.join(restored, "result.bin"))).digest("hex")) + .toBe(createHash("sha256").update(await fs.readFile(path.join(source, "result.bin"))).digest("hex")); + expect((await git(restored, "show", ":result.bin")).stdout).toBe((await git(source, "show", ":result.bin")).stdout); + expect(await fs.readFile(path.join(restored, "untracked.txt"), "utf8")).toBe("final dirty workspace bytes\n"); + expect(await fs.readlink(path.join(restored, "result-link"))).toBe("untracked.txt"); + await git(restored, "fsck", "--full"); + expect(await readWorkspaceInventory(source)).toEqual(before); +}, 30_000); + + +it.each(["separate_capture", "inventory_expired", "capture_deadline", "capture_cancelled", "final_inventory_expired"] as const)( + "keeps read-only capture phase budgets and source safety: %s", async scenario => { + const before = await readWorkspaceInventory(source); + let advancedMs = 0; + vi.spyOn(performance, "now").mockImplementation(() => advancedMs); + const deadlines: Array<{ ms: number; controller: AbortController }> = []; + vi.spyOn(AbortSignal, "timeout").mockImplementation(ms => { + const controller = new AbortController(); + deadlines.push({ ms, controller }); + return controller.signal; + }); + const actualCapture = archiveOwner.captureRemoteWorkspaceDirectory; + let archiveCompleted = false; + const capture = vi.spyOn(archiveOwner, "captureRemoteWorkspaceDirectory").mockImplementation(async input => { + expect(deadlines[0]!.ms).toBe(120_000); + expect(deadlines[1]!.ms).toBe(300_000); + expect(input.signal).toBe(deadlines[1]!.controller.signal); + advancedMs += 5_000; + deadlines[0]!.controller.abort(new DOMException("Metadata timeout", "TimeoutError")); + let interruption: ReturnType | undefined; + if (scenario === "capture_deadline" || scenario === "capture_cancelled") { + interruption = setTimeout(() => { + advancedMs += 300_000; + deadlines[1]!.controller.abort(new DOMException("Capture interrupted", + scenario === "capture_deadline" ? "TimeoutError" : "AbortError")); + }, 10); + } + try { + const result = await actualCapture(input); + archiveCompleted = true; + return result; + } finally { if (interruption) clearTimeout(interruption); } + }); + let inventoryCalls = 0; + let copiedRoot = ""; + const execute: CommandManagedRuntimeRunner["execute"] = async input => { + if (!input.args?.[1]?.includes("const inspect=")) return { exitCode: 1, stdout: "", + stderr: REMOTE_WORKSPACE_SANDBOX_DISK_FULL, timedOut: false, signal: null, + pid: process.pid, startedAt: new Date().toISOString() }; + const result = await exec(input.command, input.args, { cwd: input.cwd, timeout: 30_000, + maxBuffer: 4 * 1024 ** 2, env: { PATH: process.env.PATH, ...input.env } }); + inventoryCalls++; + if (inventoryCalls === 2) { + advancedMs += 118_000; + if (scenario === "inventory_expired") deadlines[0]!.controller.abort(new DOMException("Metadata timeout", "TimeoutError")); + } + if (inventoryCalls === 3) { + expect(input.timeoutMs).toBe(2_000); + if (scenario === "final_inventory_expired") { + advancedMs += 2_000; + deadlines[2]!.controller.abort(new DOMException("Metadata timeout", "TimeoutError")); + } + } + return { ...result, exitCode: 0, timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }; + }; + const syncOut: NonNullable = async operations => { + for (const operation of operations) for (const file of operation.files) { + copiedRoot = file.targetPath; + await fs.cp(file.sourcePath, file.targetPath, { recursive: true, verbatimSymlinks: true, + filter: current => !file.exclude?.some(excluded => { + const relative = path.relative(file.sourcePath, current).split(path.sep).join("/"); + return relative === excluded || relative.startsWith(`${excluded}/`); + }) }); + } + return { operations: operations.map(operation => ({ operationId: operation.operationId, + filesTransferred: operation.files.length, bytesTransferred: before.byteSize })) }; + }; + const target: RemoteWorkspaceTarget = { kind: "remote", transport: "sandbox", remoteCwd: source, + runner: { execute, syncOut, syncIn: syncOut } }; + let failure: unknown; + const saved = await captureRemoteWorkspaceArchive({ target, remoteCwd: source, operationId: randomUUID() }) + .catch(error => { failure = error; return null; }); + try { + expect(await readWorkspaceInventory(source)).toEqual(before); + if (scenario === "separate_capture") { + expect(failure).toBeUndefined(); + expect(saved).not.toBeNull(); + expect(archiveCompleted).toBe(true); + expect(inventoryCalls).toBe(3); + expect(deadlines.map(item => item.ms).slice(0, 2)).toEqual([120_000, 300_000]); + const restored = path.join(root, "restored-fallback"); + await archiveOwner.restoreRemoteWorkspaceArchive({ archivePath: saved!.archivePath, + targetDir: restored, expectedManifestDigest: saved!.manifestDigest }); + expect(await fs.readFile(path.join(restored, "untracked.txt"), "utf8")).toBe("final dirty workspace bytes\n"); + expect(await fs.readlink(path.join(restored, "result-link"))).toBe("untracked.txt"); + expect((await git(restored, "rev-parse", "HEAD")).stdout).toBe((await git(source, "rev-parse", "HEAD")).stdout); + expect((await git(restored, "show", ":result.bin")).stdout).toBe((await git(source, "show", ":result.bin")).stdout); + await git(restored, "fsck", "--full"); + } else { + expect(saved).toBeNull(); + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toBe(REMOTE_WORKSPACE_SANDBOX_DISK_FULL); + const stage = scenario === "inventory_expired" || scenario === "final_inventory_expired" ? "verify" : "archive"; + expect((failure as Error).cause).toMatchObject({ message: expect.stringContaining(`stage=${stage} reason=operation_failed`) }); + expect(await fs.lstat(copiedRoot).catch(() => null)).toBeNull(); + if (scenario === "inventory_expired") expect(capture).not.toHaveBeenCalled(); + if (scenario === "final_inventory_expired") expect(archiveCompleted).toBe(true); + if (scenario.startsWith("capture_")) expect(archiveCompleted).toBe(false); + } + } finally { await saved?.cleanup(); } +}, 30_000); diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index a4a3ec68f51..f46734cca37 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -203,8 +203,10 @@ export function sandboxProviderFailureLabel(error: unknown): { label: string; co return { label, code, status }; } +const WORKSPACE_COMMAND_TIMEOUT_MS = 300_000; + async function command(target: RemoteWorkspaceTarget, operation: RemoteWorkspaceOperation, - args: string[], env?: Record, timeoutMs = 300_000) { + args: string[], env?: Record, timeoutMs = WORKSPACE_COMMAND_TIMEOUT_MS) { const execute = () => target.runner!.execute({ command: "node", args, cwd: target.remoteCwd, env: { NODE_OPTIONS: "", NODE_PATH: "", ...env }, timeoutMs, bypassSession: true }); // Copy-back and restore record why the provider refused the command. @@ -466,7 +468,7 @@ async function captureReadOnlyWorkspaceArchive(input: { target: RemoteWorkspaceTarget; remoteCwd: string; operationId: string; }) { if (!/^[a-f0-9-]{36}$/.test(input.operationId)) throw new Error("remote_workspace_operation_identity_invalid"); - const inspect = async (timeoutMs = 300_000) => { + const inspect = async (timeoutMs = WORKSPACE_COMMAND_TIMEOUT_MS) => { const value = JSON.parse(await command(input.target, "capture", ["-e", readonlyWorkspaceInventoryScript(), input.remoteCwd], undefined, timeoutMs)) as Awaited>; if (!/^[a-f0-9]{64}$/.test(value.sha256) || !Number.isSafeInteger(value.byteSize) || value.byteSize < 0 || !Number.isSafeInteger(value.entries) || value.entries < 0 || !Array.isArray(value.exclude) @@ -495,11 +497,15 @@ async function captureReadOnlyWorkspaceArchive(input: { const same = (observed: typeof before) => observed.sha256 === before.sha256 && observed.byteSize === before.byteSize && observed.entries === before.entries; if (!same(received) || !same(await inspect(remaining()))) throw new Error("remote_workspace_readonly_capture_changed"); + const finalVerificationBudget = remaining(); stage = "archive"; - const receipt = await captureRemoteWorkspaceDirectory({ cwd: copy, archivePath, exclude: before.exclude, signal: verificationSignal }); + const captureSignal = AbortSignal.timeout(WORKSPACE_COMMAND_TIMEOUT_MS); + const receipt = await captureRemoteWorkspaceDirectory({ cwd: copy, archivePath, exclude: before.exclude, signal: captureSignal }); + captureSignal.throwIfAborted(); stage = "verify"; - if (!same(await inspect(remaining()))) throw new Error("remote_workspace_readonly_capture_changed"); - verificationSignal.throwIfAborted(); + const finalVerificationSignal = AbortSignal.timeout(finalVerificationBudget); + if (!same(await inspect(finalVerificationBudget))) throw new Error("remote_workspace_readonly_capture_changed"); + finalVerificationSignal.throwIfAborted(); // Only the sealed portable archive remains a candidate for durable upload. await removeTaskDirectoryTree(copy); const savedRoot = root; diff --git a/tests/e2e/onboarding.spec.ts b/tests/e2e/onboarding.spec.ts index 8960ce99819..33fba2c7f37 100644 --- a/tests/e2e/onboarding.spec.ts +++ b/tests/e2e/onboarding.spec.ts @@ -240,6 +240,18 @@ test.describe("Onboarding wizard", () => { await route.fulfill({ response }); }); + let persistedAgentId = ""; + let persistedAgentStatus = 0; + await page.route("**/api/agents/*", async route => { + if (route.request().method() !== "PATCH") return route.continue(); + const body = route.request().postDataJSON(); + if (body.defaultEnvironmentId !== FAKE_SANDBOX_ENVIRONMENT_ID) return route.continue(); + const response = await route.fetch({ postData: JSON.stringify({ ...body, defaultEnvironmentId: null }) }); + persistedAgentId = new URL(route.request().url()).pathname.split("/").at(-1)!; + persistedAgentStatus = response.status(); + await route.fulfill({ response }); + }); + // The sign-in Connect now starts. Stubbed so the card is deterministic: // the session start answers, and the guarded prompt read hands back an // authorization URL for the card's link row. @@ -346,6 +358,10 @@ test.describe("Onboarding wizard", () => { const cardLink = page.getByRole("link", { name: /^Sign in to / }); await expect(cardLink).toBeVisible({ timeout: 15_000 }); await expect(cardLink).toHaveAttribute("href", /claude\.ai\/oauth\/authorize/); + await expect.poll(() => persistedAgentStatus).toBe(200); + const persisted = await page.request.get(`/api/agents/${persistedAgentId}`); + expect(persisted.ok()).toBe(true); + expect(await persisted.json()).toMatchObject({ id: persistedAgentId, name: "Ada", defaultEnvironmentId: null }); await expect( page.getByRole("button", { name: /^Sign in to / }), ).toBeEnabled({ timeout: 15_000 });