diff --git a/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts b/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts index d5f9cb6e829..8b83e3e94be 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts @@ -25,7 +25,7 @@ vi.mock("../workspace-operations.js", () => ({ workspaceOperationService: () => createRecorder: () => ({ recordOperation: async (input: { run: () => Promise }) => input.run() }), }) })); import { resumeNativeWorkspaceFinalization } from "./native-workspace-finalizer.js"; -import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL } from "../remote-workspace-transport.js"; +import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL, RemoteWorkspaceOperationError } from "../remote-workspace-transport.js"; function fixtureDb(workspace?: Record): Db { const responses = [[{ @@ -87,11 +87,21 @@ describe("native remote workspace save failures", () => { const result = await resumeNativeWorkspaceFinalization({ db: fixtureDb(workspace), runId: "run", environmentRuntime: {} as never }); expect(sync.save).toHaveBeenCalledOnce(); expect(result).toMatchObject({ status: "failed", exitCode: 1, - stderr: `workspace_sync_out_failed\n${REMOTE_WORKSPACE_SANDBOX_DISK_FULL}\n`, metadata: { workspaceSync: { code: "workspace_sync_out_failed" } } }); + stderr: `workspace_sync_out_failed\n${REMOTE_WORKSPACE_SANDBOX_DISK_FULL}\n`, + metadata: { workspaceSync: { code: "workspace_sync_out_failed", cause: REMOTE_WORKSPACE_SANDBOX_DISK_FULL } } }); + }); + + it("keeps the redacted remote operation cause on retried copy-back", async () => { + const cause = "remote_workspace_operation_failed: operation=capture exit=1; " + + "remote_workspace_capture_failed: stage=archive exit=128 program=git helper_exit=1"; + sync.save.mockRejectedValueOnce(new RemoteWorkspaceOperationError(cause)); + const result = await resumeNativeWorkspaceFinalization({ db: fixtureDb(workspace), runId: "run", environmentRuntime: {} as never }); + expect(result).toMatchObject({ status: "failed", exitCode: 1, stderr: `workspace_sync_out_failed\n${cause}\n`, + metadata: { workspaceSync: { code: "workspace_sync_out_failed", cause } } }); }); it("keeps other save failures to the bare code", async () => { - sync.save.mockRejectedValueOnce(new Error("remote_workspace_operation_failed")); + sync.save.mockRejectedValueOnce(new Error("remote_workspace_operation_failed: SYNTHETIC_PRIVATE_PATH")); const result = await resumeNativeWorkspaceFinalization({ db: fixtureDb(workspace), runId: "run", environmentRuntime: {} as never }); expect(result).toMatchObject({ status: "failed", exitCode: 1, stderr: "workspace_sync_out_failed\n" }); }); diff --git a/server/src/services/native-runtime/native-workspace-finalizer.ts b/server/src/services/native-runtime/native-workspace-finalizer.ts index f7528ce82a2..2024972a15e 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.ts @@ -1,6 +1,6 @@ import { classifyNativeWorkspaceFailure } from "./native-workspace-failure.js"; import { readRemoteWorkspaceAuthority, saveRemoteWorkspace } from "../remote-workspaces.js"; -import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL } from "../remote-workspace-transport.js"; +import { remoteWorkspaceFailureCause } from "../remote-workspace-transport.js"; import { recordNativeFinalizationFailure } from "./native-run-finalizer.js"; import { inspectPluginWorkspaceForFinalization } from "./native-plugin-workspace-finalization.js"; import type { WorkspaceWorkers } from "../plugin-workspaces.js"; @@ -45,7 +45,7 @@ function workspaceSyncFailure( status: "failed" as const, exitCode: 1, stderr: cause ? `${code}\n${cause}\n` : `${code}\n`, - metadata: { workspaceSync: { code } }, + metadata: { workspaceSync: cause ? { code, cause } : { code } }, }; } @@ -192,11 +192,11 @@ export async function attemptNativeWorkspaceFinalization(input: { metadata: { remoteWorkspace: saved } }; } catch (error) { await ownership.assertHeld(); - // The code keeps the retry policy. A full sandbox disk is named on the - // operation, because no retry can save the workspace until space is freed. - const diskFull = error instanceof Error && error.message === REMOTE_WORKSPACE_SANDBOX_DISK_FULL; + // The code keeps the retry policy. The redacted remote cause (for + // example a full sandbox disk or the failed capture step) is named on + // the operation; other error text stays off the record. return workspaceSyncFailure(classifyNativeWorkspaceFailure(error).code === "workspace_sync_out_unrecoverable" - ? "workspace_sync_out_unrecoverable" : "workspace_sync_out_failed", diskFull ? REMOTE_WORKSPACE_SANDBOX_DISK_FULL : undefined); + ? "workspace_sync_out_unrecoverable" : "workspace_sync_out_failed", remoteWorkspaceFailureCause(error) ?? undefined); } } if (nativeWorkspaceSync) { diff --git a/server/src/services/remote-workspace-snapshot-archive.test.ts b/server/src/services/remote-workspace-snapshot-archive.test.ts index e194f6624eb..e436873ce36 100644 --- a/server/src/services/remote-workspace-snapshot-archive.test.ts +++ b/server/src/services/remote-workspace-snapshot-archive.test.ts @@ -364,11 +364,11 @@ describe("portable remote workspace archives", () => { expect(await fs.readFile(archivePath, "utf8")).toBe("preexisting archive bytes\n"); expect(await fs.readFile(path.join(cwd, "saved.txt"), "utf8")).toBe("retained source bytes\n"); }); - it("keeps an unrecognized real filesystem error private in the serialized capture helper", async () => { + it("names only the code of an unrecognized real filesystem error in the serialized capture helper", async () => { const helper = path.join(root, "archive.cjs"), archivePath = path.join(root, "snapshot.pcws"); await fs.writeFile(helper, remoteWorkspaceArchiveScript()); await expect(execute(process.execPath, [helper, "capture", path.join(root, "SYNTHETIC_PRIVATE_NAME".repeat(20)), archivePath], { env })) - .rejects.toMatchObject({ code: 1, stdout: "", stderr: "remote_workspace_archive_failed" }); + .rejects.toMatchObject({ code: 1, stdout: "", stderr: "remote_workspace_archive_failed: code=ENAMETOOLONG syscall=lstat" }); await expect(fs.access(archivePath)).rejects.toMatchObject({ code: "ENOENT" }); }); it("preserves shallow repository boundaries and a nested managed checkout", async () => { diff --git a/server/src/services/remote-workspace-snapshot-archive.ts b/server/src/services/remote-workspace-snapshot-archive.ts index a3cdc4d584a..3d99ab82117 100644 --- a/server/src/services/remote-workspace-snapshot-archive.ts +++ b/server/src/services/remote-workspace-snapshot-archive.ts @@ -459,8 +459,34 @@ export function remoteWorkspaceCaptureFailureMessage(error: unknown): string | n return null; } +/** + * A bounded cause for an unclassified sandbox failure. It runs in the sandbox, + * so it is self-contained. Each field has a fixed key and a value of at most 64 + * `[A-Za-z0-9_]` characters taken from structured error properties; it never + * includes error text, command lines, or paths. + */ +export function remoteWorkspaceFailureDetail(error: unknown): string { + if (!error || typeof error !== "object") return ""; + const failure = error as { code?: unknown; signal?: unknown; syscall?: unknown; killed?: unknown; cmd?: unknown; name?: unknown; message?: unknown }; + const fields: string[] = []; + if (typeof failure.code === "string" && /^(E[A-Z0-9]{1,20}|ERR_[A-Z0-9_]{1,56})$/.test(failure.code)) fields.push(`code=${failure.code}`); + if (typeof failure.code === "number" && Number.isSafeInteger(failure.code) && failure.code >= 0 && failure.code <= 255) fields.push(`exit=${failure.code}`); + if (typeof failure.signal === "string" && /^SIG[A-Z0-9]{1,12}$/.test(failure.signal)) fields.push(`signal=${failure.signal}`); + if (typeof failure.syscall === "string" && /^[a-z_]{1,24}$/.test(failure.syscall)) fields.push(`syscall=${failure.syscall}`); + if (failure.killed === true) fields.push("killed=true"); + if (typeof failure.cmd === "string" && failure.cmd.startsWith("git ")) fields.push("program=git"); + if (typeof failure.name === "string" && ["TypeError", "RangeError", "SyntaxError", "ReferenceError", "AbortError"].includes(failure.name)) { + fields.push(`name=${failure.name}`); + } + const reason = typeof failure.message === "string" ? /^(?:remote_workspace_archive_)?([a-z_]{1,48})$/.exec(failure.message)?.[1] : undefined; + if (reason && (failure.message !== reason || ["unsafe", "unsafe_capture", "invalid_capture", "capture_missing"].includes(reason))) { + fields.push(`reason=${reason}`); + } + return fields.join(" "); +} + export function remoteWorkspaceArchiveScript(): string { // tsx's function-name preservation helper may appear in Function#toString. const source = workspaceArchive.toString().replaceAll("__vite_ssr_dynamic_import__", "import"); - return `const __name = (value) => value;\nconst captureFailure = ${remoteWorkspaceCaptureFailureMessage.toString()};\nconst archive = ${source};\nconst [kind, first, second, digest] = process.argv.slice(2);\narchive(kind === 'capture' ? {kind,cwd:first,archivePath:second} : {kind,archivePath:first,targetDir:second,expectedManifestDigest:digest}).then(value=>process.stdout.write(JSON.stringify(value))).catch(error=>{process.stderr.write(kind === 'capture' ? captureFailure(error) ?? 'remote_workspace_archive_failed' : error.message);process.exitCode=1;});\n`; + return `const __name = (value) => value;\nconst captureFailure = ${remoteWorkspaceCaptureFailureMessage.toString()};\nconst failureDetail = ${remoteWorkspaceFailureDetail.toString()};\nconst archive = ${source};\nconst [kind, first, second, digest] = process.argv.slice(2);\narchive(kind === 'capture' ? {kind,cwd:first,archivePath:second} : {kind,archivePath:first,targetDir:second,expectedManifestDigest:digest}).then(value=>process.stdout.write(JSON.stringify(value))).catch(error=>{const detail = failureDetail(error);process.stderr.write(kind === 'capture' ? captureFailure(error) ?? ('remote_workspace_archive_failed' + (detail ? ': ' + detail : '')) : error.message);process.exitCode=1;});\n`; } diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index 9e71a52946a..f40d7230ebc 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -393,7 +393,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }; execute.mockResolvedValueOnce(failure); await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) - .rejects.toThrow(/^remote_workspace_operation_failed$/); + .rejects.toThrow(/^remote_workspace_operation_failed: operation=[a-z_]+ exit=1$/); execute.mockResolvedValueOnce({ ...failure, timedOut: true, stderr: gitOperation }); await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) .rejects.toThrow(/^remote_workspace_operation_timed_out: archive_helper \(configured timeout 300000ms\)$/); @@ -435,7 +435,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { startedAt: new Date().toISOString(), stderr: "remote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace\nSYNTHETIC_PRIVATE_PATH" }); await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) - .rejects.toThrow(/^remote_workspace_operation_failed$/); + .rejects.toThrow(/^remote_workspace_operation_failed: operation=[a-z_]+ exit=1$/); }); it.each(["split", "one-shot"] as const)("preserves a Git child disk failure with %s output without its private text", async outputMode => { @@ -467,7 +467,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { execute.mockResolvedValueOnce({ exitCode: 1, stdout, stderr, timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }); await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) - .rejects.toThrow(/^remote_workspace_operation_failed$/); + .rejects.toThrow(/^remote_workspace_operation_failed: operation=[a-z_]+ exit=1$/); expect(execute).toHaveBeenCalledTimes(1); }); @@ -489,7 +489,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { return restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: path.join(scratch, "restored"), archivePath, manifestDigest: "a".repeat(64), restorationIdentity: "b".repeat(64) }); })(); - await expect(request).rejects.toThrow(/^remote_workspace_operation_failed$/); + await expect(request).rejects.toThrow(/^remote_workspace_operation_failed: operation=[a-z_]+ exit=1$/); expect(execute).toHaveBeenCalledTimes(operation === "restore" ? 2 : 1); }); @@ -512,6 +512,42 @@ describe("remote workspace transport safety with real sandbox commands", () => { .rejects.toThrow(/^remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace$/); }); + it.each(["split", "one-shot"] as const)("names the failing capture step without its private text with %s output", async outputMode => { + await git(workspace, "init", "--initial-branch=main"); + await fs.writeFile(path.join(workspace, "saved.txt"), "retained source bytes\n"); + const bin = path.join(scratch, "controlled-bin"); await fs.mkdir(bin); + await fs.writeFile(path.join(bin, "git"), + "#!/bin/sh\nprintf '%s' 'fatal: SYNTHETIC_PRIVATE_GIT_PATH/secret.txt' >&2\nexit 128\n", { mode: 0o700 }); + const { remote } = target({ PATH: `${bin}${path.delimiter}${process.env.PATH}` }, outputMode); + const failure = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }) + .then(() => null, (error: Error) => error); + expect(failure?.message).toBe("remote_workspace_operation_failed: operation=capture exit=1; " + + "remote_workspace_capture_failed: stage=archive exit=128 program=git helper_exit=1"); + expect(failure?.message).not.toContain("SYNTHETIC"); + expect(await fs.readFile(path.join(workspace, "saved.txt"), "utf8")).toBe("retained source bytes\n"); + }); + + it("names a remote command that ended by signal", async () => { + const { remote, execute } = target(); + execute.mockResolvedValueOnce({ exitCode: null as unknown as number, stdout: "", stderr: "", timedOut: false, signal: "SIGKILL", + pid: process.pid, startedAt: new Date().toISOString() }); + await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) + .rejects.toThrow(/^remote_workspace_operation_failed: operation=archive_helper signal=SIGKILL$/); + }); + + it.each([ + "remote_workspace_capture_failed: stage=archive reason=SYNTHETIC/private/path", + "remote_workspace_capture_failed: stage=archive exit=128\nSYNTHETIC_PRIVATE", + "remote_workspace_capture_failed: stage=archive token=SYNTHETIC_PRIVATE", + "SYNTHETIC_PRIVATE remote_workspace_capture_failed: stage=archive exit=1", + ])("drops a sandbox cause outside the fixed grammar: %j", async stderr => { + const { remote, execute } = target(); + execute.mockResolvedValueOnce({ exitCode: 1, stdout: "", stderr, timedOut: false, signal: null, + pid: process.pid, startedAt: new Date().toISOString() }); + await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) + .rejects.toThrow(/^remote_workspace_operation_failed: operation=archive_helper exit=1$/); + }); + it("captures and restores verified bytes with one-shot combined output", async () => { await fs.writeFile(path.join(workspace, "saved.txt"), "saved synthetic one-shot bytes\n"); const { remote, execute } = target({}, "one-shot"); diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 3b89333275b..b243095e40d 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -5,7 +5,7 @@ import os from "node:os"; import path from "node:path"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import type { RemoteWorkspaceSource } from "@paperclipai/shared"; -import { remoteWorkspaceArchiveScript, remoteWorkspaceCaptureFailureMessage } from "./remote-workspace-snapshot-archive.js"; +import { remoteWorkspaceArchiveScript, remoteWorkspaceCaptureFailureMessage, remoteWorkspaceFailureDetail } from "./remote-workspace-snapshot-archive.js"; import type { GitRemoteAuthProvider } from "./git-credentials.js"; export type RemoteWorkspaceTarget = Extract; @@ -26,7 +26,10 @@ export const REMOTE_WORKSPACE_SANDBOX_DISK_FULL = "remote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace"; // Node reports ENOSPC or EDQUOT. Git and tar child processes report the text. const DISK_FULL_CHECK = String.raw`const diskFull=e=>['ENOSPC','EDQUOT'].includes(e?.code)||/No space left on device|Disk quota exceeded/i.test(String(e?.stderr??'')+String(e?.message??''));`; -const CAPTURE_FAILURE_CHECK = `const captureFailure=${remoteWorkspaceCaptureFailureMessage.toString()};`; +const CAPTURE_FAILURE_CHECK = `const captureFailure=${remoteWorkspaceCaptureFailureMessage.toString()};const failureDetail=${remoteWorkspaceFailureDetail.toString()};`; +// The only sandbox text a capture failure may add: a fixed stage plus bounded +// structured fields from remoteWorkspaceFailureDetail. Anything else is dropped. +const SANDBOX_CAPTURE_CAUSE = /^remote_workspace_capture_failed: stage=(?:helper|archive|publish)(?: (?:helper_)?(?:code|exit|signal|syscall|killed|program|name|reason)=[A-Za-z0-9_]{1,64}){0,16}$/; type RemoteWorkspaceOperation = "prepare" | "repository" | "archive_helper" | "capture" | "restore" | "cleanup"; @@ -37,6 +40,27 @@ export class RemoteWorkspaceOperationTimeoutError extends Error { } } +/** + * A failed remote workspace command. Its message names only server-validated + * fields and an optional sandbox cause in a fixed grammar; never command output. + */ +export class RemoteWorkspaceOperationError extends Error { + constructor(message: string) { + super(message); + this.name = "RemoteWorkspaceOperationError"; + } +} + +/** + * The server-built diagnostic for a remote workspace failure, or null when the + * error text is not one of these redacted forms and must stay off the record. + */ +export function remoteWorkspaceFailureCause(error: unknown): string | null { + if (error instanceof RemoteWorkspaceOperationError || error instanceof RemoteWorkspaceOperationTimeoutError) return error.message; + if (!(error instanceof Error)) return null; + return remoteWorkspaceCaptureFailureMessage({ message: error.message }) === error.message ? error.message : null; +} + export const REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE = "remote_workspace_prepare_timeout"; export class RemoteWorkspacePreparationTimeoutError extends RemoteWorkspaceOperationTimeoutError { @@ -62,7 +86,14 @@ async function command(target: RemoteWorkspaceTarget, operation: RemoteWorkspace const reported = output.find(value => [gitOperation, REMOTE_WORKSPACE_SANDBOX_DISK_FULL].includes(value)) ?? (operation === "capture" || operation === "archive_helper" ? output.find(value => remoteWorkspaceCaptureFailureMessage({ message: value }) === value) : undefined); - throw new Error(reported ?? "remote_workspace_operation_failed"); + if (reported) throw new Error(reported); + const cause = operation === "capture" || operation === "archive_helper" + ? output.find(value => value.length <= 1024 && SANDBOX_CAPTURE_CAUSE.test(value)) : undefined; + const status = [`operation=${operation}`, + Number.isSafeInteger(result.exitCode) ? `exit=${result.exitCode}` : null, + typeof result.signal === "string" && /^SIG[A-Z0-9]{1,12}$/.test(result.signal) ? `signal=${result.signal}` : null, + ].filter(Boolean).join(" "); + throw new RemoteWorkspaceOperationError(`remote_workspace_operation_failed: ${status}${cause ? `; ${cause}` : ""}`); } return result.stdout; } @@ -208,7 +239,8 @@ async function installArchiveHelper(target: RemoteWorkspaceTarget, remoteCwd: st // A failed write can leave partial bytes. Remove only the temporary file this run created. try{await fs.writeFile(temp,Buffer.from(body,'base64'),{mode:0o600,flag:'wx'});await fs.rename(temp,file);} catch(e){if(e?.code!=='EEXIST')await fs.rm(temp,{force:true}).catch(()=>{});throw e;} - })().catch(error=>{const message=diskFull(error)?${JSON.stringify(REMOTE_WORKSPACE_SANDBOX_DISK_FULL)}:captureFailure(error);if(message)process.stderr.write(message);process.exitCode=1});`; + })().catch(error=>{const detail=failureDetail(error),message=diskFull(error)?${JSON.stringify(REMOTE_WORKSPACE_SANDBOX_DISK_FULL)} + :captureFailure(error)??'remote_workspace_capture_failed: stage=helper'+(detail?' '+detail:'');process.stderr.write(message);process.exitCode=1});`; await command(target, "archive_helper", ["-e", install, remoteCwd, directory, helper, Buffer.from(remoteWorkspaceArchiveScript()).toString("base64")]); return { directory, helper, archive: path.posix.join(directory, "workspace.pcws"), receipt: path.posix.join(directory, "receipt.json") }; } @@ -262,7 +294,13 @@ export async function captureRemoteWorkspaceArchive(input: { const gitOperation='remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace'; process.stderr.write(typeof error?.stderr==='string'&&error.stderr.trim()===gitOperation?gitOperation :diskFull(error)?${JSON.stringify(REMOTE_WORKSPACE_SANDBOX_DISK_FULL)} - :captureFailure({message:typeof error?.stderr==='string'?error.stderr.trim():null})??captureFailure(error)??'remote_workspace_capture_failed');process.exitCode=1; + :captureFailure({message:typeof error?.stderr==='string'?error.stderr.trim():null})??captureFailure(error)??(()=>{ + // The archive child reports its own bounded detail; this process adds the child's exit. + const helper=typeof error?.cmd==='string',own=failureDetail(error); + const child=helper&&typeof error.stderr==='string'?/^remote_workspace_archive_failed: ([A-Za-z0-9_= ]{1,512})$/.exec(error.stderr.trim())?.[1]:undefined; + return 'remote_workspace_capture_failed: stage='+(helper?'archive':'publish')+(child?' '+child:'') + +(own?' '+(helper?own.split(' ').map(field=>'helper_'+field).join(' '):own):''); + })());process.exitCode=1; });`; const { layout, ...receipt } = JSON.parse(await command(input.target, "capture", ["-e", capture, staged.helper, input.remoteCwd, staged.directory])) as { manifestDigest: string; sha256: string; byteSize: number; entries: number; layout: "legacy" | "captured";