diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 78621549793..0c9e4eb43f5 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1848,6 +1848,17 @@ host, sandbox, or Git remote. Files changed since the last save still depend on the live sandbox: losing that allocation can lose those edits. There is no periodic workspace backup in this release. +Remote workspace command timeouts report `remote_workspace_operation_timed_out` +with the operation (`prepare`, `repository`, `archive_helper`, `capture`, `restore`, +or `cleanup`) and its timeout in milliseconds. Command output is excluded from +that diagnostic. A timeout does not prove the remote process stopped or that a +save failed before publication. Reconcile the existing workspace and operation +receipts before recovery; the transport does not automatically repeat the command. +Export cleanup keeps this typed timeout diagnostic in the operation's +`metadata.exportCleanup.lastError`. Other cleanup errors retain a generic message. +The saved snapshot stays acknowledged and the cleanup obligation stays pending +until the existing reconciliation path settles it. + Files reads use the current committed saved version when remote authority is clean and has no active writer, even while its old allocation remains recorded. The saved loader verifies the company, workspace, snapshot, and archive before diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index 4b1d2f79987..88b13a85991 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -308,7 +308,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { .rejects.toThrow(/^remote_workspace_operation_failed$/); execute.mockResolvedValueOnce({ ...failure, timedOut: true, stderr: gitOperation }); await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) - .rejects.toThrow(/^remote_workspace_operation_failed$/); + .rejects.toThrow(/^remote_workspace_operation_timed_out: archive_helper \(configured timeout 300000ms\)$/); }); it.each([ @@ -347,6 +347,35 @@ describe("remote workspace transport safety with real sandbox commands", () => { .rejects.toThrow(/^remote_workspace_operation_failed$/); }); + it("identifies a timed-out preparation without exposing transport output or retrying it", async () => { + const { remote, execute } = target(); + const resolveGitAuth = vi.fn(async () => null); + execute.mockResolvedValueOnce({ exitCode: 0, stdout: '{"cloneRequired":true}', stderr: "", + timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }); + execute.mockResolvedValueOnce({ exitCode: null, stdout: "SYNTHETIC_PRIVATE_STDOUT", stderr: "SYNTHETIC_PRIVATE_STDERR", + timedOut: true, signal: null, pid: process.pid, startedAt: new Date().toISOString() }); + + await expect(prepareRemoteWorkspaceDirectory({ target: remote, remoteCwd: path.join(workspace, "clone"), + source: { kind: "git", repoUrl: "https://example.invalid/source.git", ref: null }, + preparationIdentity: "a".repeat(64), resolveGitAuth })) + .rejects.toThrow(/^remote_workspace_operation_timed_out: prepare \(configured timeout 300000ms\)$/); + expect(execute).toHaveBeenCalledTimes(2); + expect(resolveGitAuth).toHaveBeenCalledTimes(1); + expect(await fs.readdir(workspace)).toEqual([]); + }); + + it("reports the cleanup deadline without treating a timeout as a completed operation", async () => { + await fs.writeFile(path.join(workspace, "saved.txt"), "preserve the workspace\n"); + const { remote, execute } = target(); + execute.mockResolvedValueOnce({ exitCode: 0, stdout: "SYNTHETIC_PRIVATE_STDOUT", stderr: "SYNTHETIC_PRIVATE_STDERR", + timedOut: true, signal: null, pid: process.pid, startedAt: new Date().toISOString() }); + + await expect(cleanupRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) + .rejects.toThrow(/^remote_workspace_operation_timed_out: cleanup \(configured timeout 10000ms\)$/); + expect(execute).toHaveBeenCalledTimes(1); + expect(await fs.readFile(path.join(workspace, "saved.txt"), "utf8")).toBe("preserve the workspace\n"); + }); + it("never follows a preexisting snapshot receipt temporary symlink", async () => { await fs.writeFile(path.join(workspace, "result.txt"), "synthetic saved result\n"); const operationId = randomUUID(); diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 8355f2e4111..794a6d500f6 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -27,12 +27,23 @@ export const REMOTE_WORKSPACE_SANDBOX_DISK_FULL = // Node reports ENOSPC or EDQUOT. Git and tar child processes report the text. const DISK_FULL_CHECK = String.raw`const diskFull=e=>['ENOSPC','EDQUOT'].includes(e?.code)||/No space left on device|Disk quota exceeded/i.test(String(e?.stderr??'')+String(e?.message??''));`; -async function command(target: RemoteWorkspaceTarget, args: string[], env?: Record, timeoutMs = 300_000) { +type RemoteWorkspaceOperation = "prepare" | "repository" | "archive_helper" | "capture" | "restore" | "cleanup"; + +export class RemoteWorkspaceOperationTimeoutError extends Error { + constructor(operation: RemoteWorkspaceOperation, timeoutMs: number) { + super(`remote_workspace_operation_timed_out: ${operation} (configured timeout ${timeoutMs}ms)`); + this.name = "RemoteWorkspaceOperationTimeoutError"; + } +} + +async function command(target: RemoteWorkspaceTarget, operation: RemoteWorkspaceOperation, + args: string[], env?: Record, timeoutMs = 300_000) { const result = await target.runner!.execute({ command: "node", args, cwd: target.remoteCwd, env: { NODE_OPTIONS: "", NODE_PATH: "", ...env }, timeoutMs, bypassSession: true }); - if (result.exitCode !== 0 || result.timedOut) { + if (result.timedOut) throw new RemoteWorkspaceOperationTimeoutError(operation, timeoutMs); + if (result.exitCode !== 0) { const gitOperation = "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace"; - const reported = result.timedOut ? "" : result.stderr.trim(); + const reported = result.stderr.trim(); throw new Error([gitOperation, REMOTE_WORKSPACE_SANDBOX_DISK_FULL].includes(reported) ? reported : "remote_workspace_operation_failed"); } return result.stdout; @@ -95,14 +106,14 @@ export async function prepareRemoteWorkspaceDirectory(input: { preparationIdentity: string; }) { if (input.source.kind === "git") { - const prior = JSON.parse(await command(input.target, ["-e", PREPARE, input.remoteCwd, + const prior = JSON.parse(await command(input.target, "prepare", ["-e", PREPARE, input.remoteCwd, JSON.stringify({ ...input.source, preparationIdentity: input.preparationIdentity, authArgs: [], authEnvKeys: [], probeOnly: true })])) as { cwd?: string; cloneRequired?: boolean }; if (prior.cwd === input.remoteCwd) return; if (prior.cloneRequired !== true) throw new Error("remote_workspace_root_changed"); } const auth = input.source.kind === "git" ? await input.resolveGitAuth(input.source.repoUrl) : null; - const output = await command(input.target, ["-e", PREPARE, input.remoteCwd, + const output = await command(input.target, "prepare", ["-e", PREPARE, input.remoteCwd, JSON.stringify({ ...input.source, preparationIdentity: input.preparationIdentity, authArgs: auth?.configArgs ?? [], authEnvKeys: Object.keys(auth?.env ?? {}) })], auth?.env); const result = JSON.parse(output) as { cwd: string }; @@ -150,7 +161,7 @@ export async function prepareRemoteRepository(input: { const branch=await git(['symbolic-ref','--short','-q','HEAD']).catch(()=>null);process.stdout.write(JSON.stringify({cwd:root,pinnedCommit:input.pinnedCommit||head,branchName:branch})); })().catch(()=>{process.stderr.write('remote_workspace_repository_failed');process.exitCode=1});`; const payload = { ...input, target: undefined, resolveGitAuth: undefined }; - const prior = JSON.parse(await command(input.target, ["-e", script, input.remoteCwd, + const prior = JSON.parse(await command(input.target, "repository", ["-e", script, input.remoteCwd, JSON.stringify({ ...payload, authArgs: [], authEnvKeys: [], probeOnly: true })])) as { cwd?: string; pinnedCommit?: string; branchName?: string | null; cloneRequired?: boolean; }; @@ -159,7 +170,7 @@ export async function prepareRemoteRepository(input: { return { cwd: prior.cwd, pinnedCommit: prior.pinnedCommit!, branchName: prior.branchName ?? null }; } const auth = await input.resolveGitAuth(input.repoUrl); - const output = await command(input.target, ["-e", script, input.remoteCwd, + const output = await command(input.target, "repository", ["-e", script, input.remoteCwd, JSON.stringify({ ...payload, authArgs: auth?.configArgs ?? [], authEnvKeys: Object.keys(auth?.env ?? {}) })], auth?.env); const value = JSON.parse(output) as { cwd: string; pinnedCommit: string; branchName: string | null }; if (value.cwd !== input.remoteCwd || !/^[a-f0-9]{40,64}$/.test(value.pinnedCommit)) throw new Error("remote_workspace_repository_receipt_invalid"); @@ -180,7 +191,7 @@ async function installArchiveHelper(target: RemoteWorkspaceTarget, remoteCwd: st try{await fs.writeFile(temp,Buffer.from(body,'base64'),{mode:0o600,flag:'wx'});await fs.rename(temp,file);} catch(e){if(e?.code!=='EEXIST')await fs.rm(temp,{force:true}).catch(()=>{});throw e;} })().catch(error=>{if(diskFull(error))process.stderr.write(${JSON.stringify(REMOTE_WORKSPACE_SANDBOX_DISK_FULL)});process.exitCode=1});`; - await command(target, ["-e", install, remoteCwd, directory, helper, Buffer.from(remoteWorkspaceArchiveScript()).toString("base64")]); + await command(target, "archive_helper", ["-e", install, remoteCwd, directory, helper, Buffer.from(remoteWorkspaceArchiveScript()).toString("base64")]); return { directory, helper, archive: path.posix.join(directory, "workspace.pcws"), receipt: path.posix.join(directory, "receipt.json") }; } @@ -197,7 +208,7 @@ export async function cleanupRemoteWorkspaceArchive(input: { if(!entry.isDirectory()||entry.isSymbolicLink())throw Error('unsafe');} await fs.rm(dir,{recursive:true,force:true}); })().catch(()=>{process.stderr.write('remote_workspace_export_cleanup_failed');process.exitCode=1});`; - await command(input.target, ["-e", script, input.remoteCwd, input.operationId], undefined, 10_000); + await command(input.target, "cleanup", ["-e", script, input.remoteCwd, input.operationId], undefined, 10_000); } export async function captureRemoteWorkspaceArchive(input: { @@ -234,7 +245,7 @@ export async function captureRemoteWorkspaceArchive(input: { process.stderr.write(typeof error?.stderr==='string'&&error.stderr.trim()===gitOperation?gitOperation :diskFull(error)?${JSON.stringify(REMOTE_WORKSPACE_SANDBOX_DISK_FULL)}:'remote_workspace_capture_failed');process.exitCode=1; });`; - const { layout, ...receipt } = JSON.parse(await command(input.target, ["-e", capture, staged.helper, input.remoteCwd, staged.directory])) as { + const { layout, ...receipt } = JSON.parse(await command(input.target, "capture", ["-e", capture, staged.helper, input.remoteCwd, staged.directory])) as { manifestDigest: string; sha256: string; byteSize: number; entries: number; layout: "legacy" | "captured"; }; if (!["legacy", "captured"].includes(layout) || !/^[a-f0-9]{64}$/.test(receipt.manifestDigest) || !/^[a-f0-9]{64}$/.test(receipt.sha256) @@ -284,7 +295,7 @@ export async function restoreRemoteWorkspaceDirectory(input: { await fs.writeFile(path.join(runtime,'workspace-restored.json'),JSON.stringify({identity,manifestDigest:digest}),{flag:'wx',mode:0o600}); if(await fs.realpath(path.dirname(root))!==path.dirname(root))throw Error('unsafe_root');await fs.rename(staged,root); })().catch(()=>{process.stderr.write('remote_workspace_restore_failed');process.exitCode=1});`; - await command(input.target, ["-e", restore, staged.helper, staged.archive, input.remoteCwd, input.manifestDigest, input.restorationIdentity]); + await command(input.target, "restore", ["-e", restore, staged.helper, staged.archive, input.remoteCwd, input.manifestDigest, input.restorationIdentity]); // Publish the restored location before cleanup can fail, stall, or lose its // reply. The owner journals this receipt and then reclaims disposable staging. return { operationId, stagingRoot: parent, cleanupPending: true }; diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index 077cb768037..e1c6e2d93c3 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -919,17 +919,28 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem expect(await preserveRemoteWorkspacePreparationFailure(db, prepared.lease)).toBeNull(); }, 60_000); - it("records and retries export cleanup after a lost reply without another upload", async () => { + it.each([ + ["a lost reply", "remote_workspace_export_cleanup_failed"], + ["a timeout", "remote_workspace_operation_timed_out: cleanup (configured timeout 10000ms)"], + ["an untrusted error", "remote_workspace_export_cleanup_failed"], + ] as const)("records and retries export cleanup after %s without another upload", async (failure, diagnostic) => { const reserved = await reserve({ kind: "empty" }), first = await target("cleanup-sandbox"); const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first }); await fs.writeFile(path.join(prepared.remoteRoot, "saved.txt"), "keep working files\n"); const sandbox = first as Extract; const execute = sandbox.runner!.execute.bind(sandbox.runner!); - let lost = false; + let lost = false, cleanupAttempts = 0; sandbox.runner!.execute = async input => { const result = await execute(input); - if (!lost && input.args?.[1]?.includes("remote_workspace_export_cleanup_failed") && result.exitCode === 0) { - lost = true; throw new Error("lost cleanup reply"); + if (input.args?.[1]?.includes("remote_workspace_export_cleanup_failed") && result.exitCode === 0) { + cleanupAttempts++; + if (lost) return result; + lost = true; + if (failure === "a timeout") return { ...result, timedOut: true, + stdout: "SYNTHETIC_PRIVATE_STDOUT", stderr: "SYNTHETIC_PRIVATE_STDERR" }; + throw new Error(failure === "an untrusted error" + ? "remote_workspace_operation_timed_out: cleanup (configured timeout 10000ms)\nSYNTHETIC_PRIVATE_STDERR" + : "lost cleanup reply"); } return result; }; @@ -940,11 +951,13 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem const [before] = await db.select().from(workspaceOperations).where(and(eq(workspaceOperations.heartbeatRunId, ids.run), eq(workspaceOperations.phase, "remote_workspace_snapshot"), eq(workspaceOperations.status, "succeeded"), eq(workspaceOperations.id, (await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, saved.snapshotId)))[0]!.sourceOperationId))); - expect(before?.metadata?.exportCleanup).toMatchObject({ state: "pending", lastError: "remote_workspace_export_cleanup_failed" }); + expect(before?.metadata?.exportCleanup).toMatchObject({ state: "pending", lastError: diagnostic }); + expect(cleanupAttempts).toBe(1); expect(await saveRemoteWorkspace(input)).toMatchObject({ snapshotId: saved.snapshotId, generation: saved.generation }); expect(upload).toHaveBeenCalledOnce(); const [after] = await db.select().from(workspaceOperations).where(eq(workspaceOperations.id, before!.id)); expect(after?.metadata?.exportCleanup).toMatchObject({ state: "cleaned", lastError: null }); + expect(cleanupAttempts).toBe(2); expect(await fs.readFile(path.join(prepared.remoteRoot, "saved.txt"), "utf8")).toBe("keep working files\n"); }, 60_000); diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index bb5849d2b55..231896c716c 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -25,7 +25,7 @@ import { canonicalExecutionUuid } from "./execution-identity.js"; import { persistActivity } from "./activity-log.js"; import { captureRemoteWorkspaceArchive, cleanupRemoteWorkspaceArchive, prepareRemoteWorkspaceDirectory, prepareRemoteRepository, remoteWorkspaceRepositoryPath, - requireRemoteWorkspaceTarget, restoreRemoteWorkspaceDirectory, + RemoteWorkspaceOperationTimeoutError, requireRemoteWorkspaceTarget, restoreRemoteWorkspaceDirectory, } from "./remote-workspace-transport.js"; import { beginWorkspaceSnapshot, getWorkspaceSnapshot, loadWorkspaceSnapshotArchive, uploadWorkspaceSnapshot, @@ -281,16 +281,19 @@ async function cleanupSavedExports(input: { db: Db; companyId: string; workspace for (const row of rows) { await input.assertOwned(); let cleaned = false; + let lastError: string | null = null; try { await cleanupRemoteWorkspaceArchive({ target: input.target, remoteCwd: row.phase === "remote_workspace_restore" ? path.posix.dirname(input.location.remoteCwd) : input.location.remoteCwd, operationId: row.id }); cleaned = true; - } catch { /* Keep a durable retry obligation; the acknowledged snapshot is safe. */ } + } catch (error) { + lastError = error instanceof RemoteWorkspaceOperationTimeoutError ? error.message : "remote_workspace_export_cleanup_failed"; + } await input.assertOwned(); await input.db.update(workspaceOperations).set({ metadata: sql`jsonb_set(coalesce(${workspaceOperations.metadata}, '{}'::jsonb), '{exportCleanup}', coalesce(${workspaceOperations.metadata}->'exportCleanup', '{}'::jsonb) || - ${JSON.stringify({ state: cleaned ? "cleaned" : "pending", lastError: cleaned ? null : "remote_workspace_export_cleanup_failed" })}::jsonb)`, + ${JSON.stringify({ state: cleaned ? "cleaned" : "pending", lastError })}::jsonb)`, updatedAt: new Date() }) .where(and(eq(workspaceOperations.id, row.id), eq(workspaceOperations.companyId, input.companyId), inArray(workspaceOperations.status, ["succeeded", "failed"])));