diff --git a/server/src/__tests__/native-workspace-finalization-work.test.ts b/server/src/__tests__/native-workspace-finalization-work.test.ts index 724a2bce425..efa36ec0ef1 100644 --- a/server/src/__tests__/native-workspace-finalization-work.test.ts +++ b/server/src/__tests__/native-workspace-finalization-work.test.ts @@ -415,4 +415,37 @@ describe("workspace finalization scheduled lifecycle", () => { await vi.advanceTimersByTimeAsync(Math.max(0, failed.nextAttemptAt!.getTime() - Date.now())); await flush(); expect(await successful(binding.runId)).toHaveLength(1); await quiet(); }); + it("names the last redacted copy-back cause on the run and coordinator after retries stop", async () => { + const binding = await seed(); await complete(binding); + const cause = "remote_workspace_operation_failed: operation=archive_helper transport=sandbox_lease_inactive"; + // The last attempt timed out; the server-built timeout text must also reach the run. + const timeout = "remote_workspace_operation_timed_out: capture (configured timeout 300000ms)"; + const causes = [cause, "remote_workspace_operation_failed: /home/SYNTHETIC/private", timeout]; + const attempt = vi.fn(async (input: Parameters[0]) => { + const [operation] = await db.insert(workspaceOperations).values({ companyId, issueId: binding.issueId, + heartbeatRunId: input.runId, phase: "workspace_finalize", status: "failed", exitCode: 1, + stderrExcerpt: "workspace_sync_out_failed\n", + metadata: { workspaceSync: { code: "workspace_sync_out_failed", cause: causes[attempt.mock.calls.length - 1] } } }).returning(); + return { kind: "completed", operation } as const; + }); + const service = nativeWorkspaceFinalizationWork(db, { attempt, onWorkspaceSettled: settled }); + for (let index = 0; index < 3; index++) { + const [entry] = await service.load({ companyId, runId: binding.runId }); + if (!entry) break; + await vi.advanceTimersByTimeAsync(Math.max(0, entry.dueAt - Date.now())); + await service.check(entry); + const [coordinatorRow] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, binding.runId)); + // A cause outside the fixed grammar never reaches the run. + expect(coordinatorRow.failureDetail?.failureCause ?? null).toBe([cause, null, timeout][index]); + } + expect(attempt).toHaveBeenCalledTimes(3); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, binding.runId)); + expect(run).toMatchObject({ status: "failed", errorCode: "native_workspace_sync_out_retry_exhausted", + resultJson: expect.objectContaining({ failureCause: timeout }) }); + expect(run.error).toContain(`Last cause: ${timeout}`); + expect(JSON.stringify(run)).not.toContain("SYNTHETIC"); + const [exhausted] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, binding.runId)); + expect(exhausted).toMatchObject({ phase: "terminal_failure", failureCode: "native_workspace_sync_out_retry_exhausted", + failureDetail: expect.objectContaining({ failureCause: timeout }) }); + }); }); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index 258a50b501f..faee8883e49 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -1,4 +1,6 @@ import { readClaimedNativeStopRecovery } from "../native-runtime/native-cancellation-request.js"; +import { remoteWorkspaceFailureCause } from "../remote-workspace-transport.js"; +import { readNativePostResultFailureDiagnostic } from "../native-runtime/native-post-result-failure.js"; import { NativeStartupCleanupPendingError } from "../native-runtime/native-startup-cleanup.js"; import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js"; import { describeFailedAttachment } from "../native-runtime/remote-warm-session-recovery.js"; @@ -1153,12 +1155,16 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe // state. Best-effort record finalize=failed so the dependent readiness // check keeps the gate closed instead of waking on stale local state, // and surface the original error to the caller. + // The control code can be a fencing label; the closed diagnostic says + // what actually failed after an accepted result (no provider text). + const postResultFailure = readNativePostResultFailureDiagnostic(adapterErr); try { await recordWorkspaceFinalize("failed", { errorMessage: adapterErr instanceof Error ? adapterErr.message : String(adapterErr), + ...(postResultFailure ? { failureDiagnostic: postResultFailure } : {}), }); } catch (recordErr) { logger.warn( @@ -1195,6 +1201,9 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe projectRunStatus: true, failureScope: "workspace", permanent: unrecoverable, + failureCause: postResultFailure + ? `native_post_result_failure: category=${postResultFailure.category}` + : remoteWorkspaceFailureCause(adapterErr), }); input.effects.onNativeWorkspaceFinalizeScheduled(); throw new NativeWorkspaceFinalizeScheduledError( diff --git a/server/src/services/native-runtime/native-post-result-failure.test.ts b/server/src/services/native-runtime/native-post-result-failure.test.ts new file mode 100644 index 00000000000..2bd2655e8a0 --- /dev/null +++ b/server/src/services/native-runtime/native-post-result-failure.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; +import { NativeSessionCloseUnrecoverableError } from "../../vendor/paperclip-runner/index.js"; +import { nativePostResultFailureCategory, readNativePostResultFailureDiagnostic } from "./native-post-result-failure.js"; + +describe("native post-result failure diagnostic", () => { + it("names a runner that kept the result but did not prove its suspension", () => { + const unsettled = new NativeSessionCloseUnrecoverableError({ runnerSuspended: false, providerDrained: false }, true); + expect(nativePostResultFailureCategory(unsettled)).toBe("runner_settlement_incomplete"); + expect(nativePostResultFailureCategory(new AggregateError([unsettled], "close and receipt failed"))) + .toBe("runner_settlement_incomplete"); + expect(nativePostResultFailureCategory(Object.assign(new Error("missing"), { code: "ENOENT" }))).toBe("io_not_found"); + expect(nativePostResultFailureCategory(new Error("SYNTHETIC other failure"))).toBe("execution_failure"); + }); + + it("reads only the closed labels from an error or its cause", () => { + const diagnostic = { stage: "post_result_execution", category: "runner_settlement_incomplete", resultAlreadyAccepted: true }; + const control = Object.assign(new Error("native_session_lease_lost"), { diagnostic }); + expect(readNativePostResultFailureDiagnostic(control)).toEqual(diagnostic); + expect(readNativePostResultFailureDiagnostic(new Error("wrapped", { cause: control }))).toEqual(diagnostic); + expect(readNativePostResultFailureDiagnostic(Object.assign(new Error("native_session_lease_lost"), + { diagnostic: { ...diagnostic, category: "/SYNTHETIC/private" } }))).toBeNull(); + expect(readNativePostResultFailureDiagnostic(new Error("native_session_lease_lost"))).toBeNull(); + }); +}); diff --git a/server/src/services/native-runtime/native-post-result-failure.ts b/server/src/services/native-runtime/native-post-result-failure.ts new file mode 100644 index 00000000000..06063f17b7e --- /dev/null +++ b/server/src/services/native-runtime/native-post-result-failure.ts @@ -0,0 +1,52 @@ +import { NativeSessionCloseUnrecoverableError } from "../../vendor/paperclip-runner/index.js"; + +/** + * Closed labels for a native execution failure after its result was accepted. + * The labels explain the failure; they grant no finalization or retry authority. + */ +export const NATIVE_POST_RESULT_FAILURE_CATEGORIES = [ + "io_not_found", "io_permission_denied", "io_failure", "runner_settlement_incomplete", "execution_failure", +] as const; + +export type NativePostResultFailureCategory = typeof NATIVE_POST_RESULT_FAILURE_CATEGORIES[number]; + +export type NativePostResultFailureDiagnostic = Readonly<{ + stage: "post_result_execution"; + category: NativePostResultFailureCategory; + resultAlreadyAccepted: true; +}>; + +function closeUnrecoverable(error: unknown): boolean { + for (let current = error, depth = 0; current && depth < 4; depth++) { + if (current instanceof NativeSessionCloseUnrecoverableError) return true; + if (current instanceof AggregateError && current.errors.some(item => item instanceof NativeSessionCloseUnrecoverableError)) return true; + current = (current as { cause?: unknown }).cause; + } + return false; +} + +/** Classify the execution error whose failure write an accepted result prevented. */ +export function nativePostResultFailureCategory(error: unknown): NativePostResultFailureCategory { + const code = error && typeof error === "object" ? (error as { code?: unknown }).code : undefined; + return code === "ENOENT" ? "io_not_found" + : code === "EACCES" || code === "EPERM" ? "io_permission_denied" + : code === "EIO" ? "io_failure" + // The runner kept the result but did not prove a durable suspension before + // shutdown. This is not an ownership loss. + : closeUnrecoverable(error) ? "runner_settlement_incomplete" + : "execution_failure"; +} + +/** Read the diagnostic from an error or its causes; only the closed labels pass. */ +export function readNativePostResultFailureDiagnostic(error: unknown): NativePostResultFailureDiagnostic | null { + for (let current = error, depth = 0; current && depth < 4; current = (current as { cause?: unknown }).cause, depth++) { + const diagnostic = (current as { diagnostic?: unknown }).diagnostic; + if (!diagnostic || typeof diagnostic !== "object") continue; + const { stage, category, resultAlreadyAccepted } = diagnostic as Record; + if (stage === "post_result_execution" && resultAlreadyAccepted === true + && (NATIVE_POST_RESULT_FAILURE_CATEGORIES as readonly unknown[]).includes(category)) { + return { stage, category: category as NativePostResultFailureCategory, resultAlreadyAccepted }; + } + } + return null; +} diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 55482228c82..d5f850982ac 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -262,11 +262,23 @@ async function claimCoordinator(input: { return claimed; } -function terminalFinalizationFailureMessage(failureCode: string, failureScope: "provider" | "workspace" | undefined) { +/** + * A workspace copy-back cause is server-built text in a fixed grammar (see + * remoteWorkspaceFailureCause and the native post-result diagnostic). Re-check the shape before it reaches the run: + * no paths, no line breaks, bounded length. + */ +export function readWorkspaceFailureCause(value: unknown): string | null { + return typeof value === "string" && value.length <= 1024 + && /^(?:remote_workspace|native_post_result)_[a-z_]+(?::[A-Za-z0-9_=;:,.() -]*)?$/.test(value) ? value : null; +} + +function terminalFinalizationFailureMessage(failureCode: string, failureScope: "provider" | "workspace" | undefined, + failureCause?: string | null) { if (failureCode === "native_workspace_sync_out_retry_exhausted") { // This code also covers local-directory and plugin workspaces, so the // message stays general and defers to the recovery action's next step. - return "Saving the workspace failed after 3 attempts. The agent's result is kept; follow the run's recovery action to repair the workspace."; + return "Saving the workspace failed after 3 attempts. The agent's result is kept; follow the run's recovery action to repair the workspace." + + (failureCause ? ` Last cause: ${failureCause}` : ""); } if (failureCode === "native_workspace_sync_out_unrecoverable") { return "The sandbox holding unexported workspace changes is unavailable, so its files could not be saved."; @@ -289,9 +301,11 @@ async function recordRetryableFailure(input: { projectRunStatus?: boolean; failureScope?: "provider" | "workspace"; permanent?: boolean; + failureCause?: string | null; }) { const now = new Date(); const nextAttemptAt = new Date(now.getTime() + 30_000); + const failureCause = readWorkspaceFailureCause(input.failureCause); const acceptedRunTerminalState = record( input.run.resultJson, ).prpRunTerminalState; @@ -435,6 +449,7 @@ async function recordRetryableFailure(input: { failureDetail: { message: input.message.slice(0, 2_000), originalFailureCode: input.failureCode, + ...(failureCause ? { failureCause } : {}), ...(priorFailureDetail.workspaceExportRetry ? { workspaceExportRetry: priorFailureDetail.workspaceExportRetry } : {}), ...(workspaceFinalizeAttempt === null ? {} @@ -476,7 +491,7 @@ async function recordRetryableFailure(input: { ...(terminalRunStatus === "failed" ? explainFailedRun({ errorCode: failureCode, - error: terminalFinalizationFailureMessage(failureCode, input.failureScope), + error: terminalFinalizationFailureMessage(failureCode, input.failureScope, failureCause), }) : {}), } @@ -489,6 +504,8 @@ async function recordRetryableFailure(input: { finalizationPhase: phase, failureCode, originalFailureCode: input.failureCode, + // The latest attempt's redacted cause; cleared when it named none. + failureCause, nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt.toISOString(), })}::jsonb`, updatedAt: now, @@ -607,6 +624,7 @@ export async function recordNativeFinalizationFailure(input: { projectRunStatus?: boolean; failureScope?: "provider" | "workspace"; permanent?: boolean; + failureCause?: string | null; }) { const [run, coordinator] = await Promise.all([ input.db @@ -647,6 +665,7 @@ export async function recordNativeFinalizationFailure(input: { projectRunStatus: input.projectRunStatus, failureScope: input.failureScope, permanent: input.permanent, + failureCause: input.failureCause, }); } diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index d1e9d671973..f55117d6b26 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,4 +1,5 @@ import { NativeSessionDiagnostics, nativeSessionDiagnostics, type NativeDiagnosticLease } from "./native-session-diagnostics.js"; +import { nativePostResultFailureCategory, type NativePostResultFailureCategory } from "./native-post-result-failure.js"; import { logger } from "../../middleware/logger.js"; import { canRestoreMigratedNativeSession } from "./native-workspace-migration.js"; import { prepareCursorCloudRuntime } from "./cursor-cloud-runtime.js"; @@ -309,8 +310,6 @@ export class NativeCancellationPendingRecoveryError extends Error { } } -type NativePostResultFailureCategory = "io_not_found" | "io_permission_denied" | "io_failure" | "execution_failure"; - class NativePostResultFailureError extends Error { readonly diagnostic: Readonly<{ stage: "post_result_execution"; @@ -322,10 +321,7 @@ class NativePostResultFailureError extends Error { // Keep the existing control classification. These labels explain why its // failure write was rejected; they grant no finalization or retry authority. super("native_session_lease_lost"); - const code = record(error).code; - const category: NativePostResultFailureCategory = code === "ENOENT" ? "io_not_found" - : code === "EACCES" || code === "EPERM" ? "io_permission_denied" - : code === "EIO" ? "io_failure" : "execution_failure"; + const category = nativePostResultFailureCategory(error); this.diagnostic = Object.freeze({ stage: "post_result_execution", category, resultAlreadyAccepted: true }); } } diff --git a/server/src/services/native-runtime/native-workspace-finalizer.ts b/server/src/services/native-runtime/native-workspace-finalizer.ts index 2024972a15e..cd0967e63be 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.ts @@ -357,7 +357,8 @@ export async function resumeNativeWorkspaceFinalization(input: Parameters | null }) { const unrecoverable = operation.stderrExcerpt?.includes("workspace_sync_out_unrecoverable"); const pluginFailure = operation.stderrExcerpt?.trim(); const pluginRepair = pluginFailure === "plugin_workspace_branch_mismatch" || pluginFailure === "plugin_workspace_owner_mismatch"; @@ -366,5 +367,6 @@ export function recordNativeWorkspaceFinalizationFailure(db: Db, runId: string, error: new Error(pluginRepair ? `native_${pluginFailure}` : unrecoverable ? "native_workspace_sync_out_unrecoverable" : "native_workspace_sync_out_failed"), projectRunStatus: true, failureScope: "workspace", permanent: unrecoverable, + failureCause: readString(record(record(operation.metadata).workspaceSync).cause), }); } diff --git a/server/src/services/remote-workspace-save-concurrency.test.ts b/server/src/services/remote-workspace-save-concurrency.test.ts index 9ecf9f04d3b..16e7293375d 100644 --- a/server/src/services/remote-workspace-save-concurrency.test.ts +++ b/server/src/services/remote-workspace-save-concurrency.test.ts @@ -486,4 +486,33 @@ describe.skipIf(!support.supported)("remote workspace save parent lock order", ( expect(snapshots.filter(snapshot => snapshot.status === "committed")).toHaveLength(1); expect(await readFile(path.join(authority.location!.remoteCwd, "saved.txt"), "utf8")).toBe("one preserved result\n"); }, 30_000); + + it("ends a failed save operation with its redacted cause and reopens it for the retry", async () => { + const f = await preparedFailure(); + const refused: AdapterSandboxExecutionTarget = { ...f.target, runner: { ...f.target.runner!, + execute: async () => { throw new Error("Daytona sandbox lease SYNTHETIC-private-id is no longer active."); } } }; + await expect(saveRemoteWorkspace({ db, storage: f.storage, workspace: f.workspace, runId: f.runId, lease: f.lease, + target: refused, assertOwned: async () => undefined })) + .rejects.toThrow("Daytona sandbox lease SYNTHETIC-private-id is no longer active."); + const snapshotOperation = async () => (await db.select().from(workspaceOperations).where(and( + eq(workspaceOperations.heartbeatRunId, f.runId), eq(workspaceOperations.phase, "remote_workspace_snapshot"), + sql`${workspaceOperations.metadata}->>'boundary' = 'final'`)))[0]!; + const failed = await snapshotOperation(); + expect(failed).toMatchObject({ status: "failed", exitCode: 1, + stderrExcerpt: "remote_workspace_operation_failed\n" + + "remote_workspace_operation_failed: operation=archive_helper transport=sandbox_lease_inactive\n", + metadata: { lastFailure: { code: "remote_workspace_operation_failed", + cause: "remote_workspace_operation_failed: operation=archive_helper transport=sandbox_lease_inactive" } } }); + expect(failed.finishedAt).toBeInstanceOf(Date); + expect(JSON.stringify(failed)).not.toContain("SYNTHETIC"); + + // The bounded retry reuses the same boundary operation and records success. + const saved = await saveRemoteWorkspace({ db, storage: f.storage, workspace: f.workspace, runId: f.runId, lease: f.lease, + target: f.target, assertOwned: async () => undefined }); + const succeeded = await snapshotOperation(); + expect(succeeded.id).toBe(failed.id); + expect(succeeded).toMatchObject({ status: "succeeded", metadata: { snapshotId: saved.snapshotId } }); + expect(await readFile(path.join(readRemoteWorkspaceAuthority(f.workspace.metadata)!.location!.remoteCwd, "saved.txt"), "utf8")) + .toBe("one preserved result\n"); + }, 30_000); }); diff --git a/server/src/services/remote-workspace-snapshot-retention.ts b/server/src/services/remote-workspace-snapshot-retention.ts index 747339eff06..d4722c07d01 100644 --- a/server/src/services/remote-workspace-snapshot-retention.ts +++ b/server/src/services/remote-workspace-snapshot-retention.ts @@ -34,6 +34,10 @@ async function pinned(db: ReadDb, row: Snapshot) { const [operation] = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( eq(workspaceOperations.companyId, row.companyId), eq(workspaceOperations.executionWorkspaceId, row.executionWorkspaceId), or(inArray(workspaceOperations.status, ["running", "pending"]), + // A failed save keeps its snapshot pinned for its retry, as while it ran, + // until the final save retires it. + sql`(${workspaceOperations.status} = 'failed' and ${workspaceOperations.metadata} ? 'lastFailure' + and ${workspaceOperations.metadata}->'retiredSave' is null)`, sql`(${workspaceOperations.metadata}->'saveWork'->>'schema' = 'paperclip.remote-workspace-save-work.v1' and (${workspaceOperations.metadata}#>'{saveWork,unknown}' is distinct from 'false'::jsonb or case when jsonb_typeof(${workspaceOperations.metadata}#>'{saveWork,activeIds}') = 'array' diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index f40d7230ebc..8bbea25b46e 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -13,9 +13,11 @@ import { cleanupRemoteWorkspaceArchive, prepareRemoteRepository, prepareRemoteWorkspaceDirectory, + remoteWorkspaceFailureCause, restoreRemoteWorkspaceDirectory, type RemoteWorkspaceTarget, } from "./remote-workspace-transport.js"; +import { classifyNativeWorkspaceFailure } from "./native-runtime/native-workspace-failure.js"; const exec = promisify(execFile); @@ -535,6 +537,61 @@ describe("remote workspace transport safety with real sandbox commands", () => { .rejects.toThrow(/^remote_workspace_operation_failed: operation=archive_helper signal=SIGKILL$/); }); + it.each([ + ["Daytona sandbox lease SYNTHETIC-sandbox-id is no longer active.", undefined, "transport=sandbox_lease_inactive"], + ["daytona_sandbox_not_found: /home/SYNTHETIC/private", undefined, "transport=daytona_sandbox_not_found"], + ["connect to /SYNTHETIC/private failed", "ECONNRESET", "transport=transport_closed code=ECONNRESET"], + ["SYNTHETIC_PRIVATE provider response", undefined, "transport=provider_error"], + ])("names a copy-back command the provider refused without its text: %j", async (message, code, named) => { + const { remote, execute } = target(); + execute.mockRejectedValueOnce(Object.assign(new Error(message), code ? { code } : {})); + const failure = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }) + .then(() => null, (error: unknown) => error as Error); + // The thrown error is unchanged for existing callers; only the recorded cause is redacted. + expect(failure?.message).toBe(message); + expect(remoteWorkspaceFailureCause(failure)).toBe(`remote_workspace_operation_failed: operation=archive_helper ${named}`); + expect(remoteWorkspaceFailureCause(failure)).not.toMatch(/SYNTHETIC|private/); + // The sandbox-loss classification that stops retries still sees its code. + expect(classifyNativeWorkspaceFailure(failure).code).toBe(message.startsWith("daytona_sandbox_not_found") + ? "workspace_sync_out_unrecoverable" : "workspace_sync_out_failed"); + }); + + it("names a failed archive download without the provider text", async () => { + await fs.writeFile(path.join(workspace, "saved.txt"), "saved synthetic bytes\n"); + const { remote } = target(); + remote.runner!.syncOut = vi.fn(async () => { throw new Error("Daytona download of /SYNTHETIC/private timed out"); }); + const failure = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }) + .then(() => null, (error: unknown) => error as Error); + expect(remoteWorkspaceFailureCause(failure)).toBe("remote_workspace_operation_failed: operation=download transport=transport_timeout"); + }); + + it("names a permission-denied archive download without its path", async () => { + await fs.writeFile(path.join(workspace, "saved.txt"), "saved synthetic bytes\n"); + const { remote } = target(); + remote.runner!.syncOut = vi.fn(async () => { + throw Object.assign(new Error("EACCES: permission denied, open '/SYNTHETIC/private'"), { code: "EACCES" }); + }); + const failure = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }) + .then(() => null, (error: unknown) => error as Error); + expect(remoteWorkspaceFailureCause(failure)) + .toBe("remote_workspace_operation_failed: operation=download transport=permission_denied code=EACCES"); + }); + + it("keeps an archive-safety refusal exact for its own retry policy", async () => { + await fs.writeFile(path.join(workspace, "saved.txt"), "saved synthetic bytes\n"); + const { remote } = target(); + const refusal = "Daytona syncOut refusing tarball member that escapes the extraction dir: ../private"; + remote.runner!.syncOut = vi.fn(async () => { throw new Error(refusal); }); + const failure = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }) + .then(() => null, (error: unknown) => error as Error); + expect(failure?.message).toBe(refusal); + expect(remoteWorkspaceFailureCause(failure)).toBeNull(); + }); + + it("does not label a provider error that did not come from a copy-back command", () => { + expect(remoteWorkspaceFailureCause(new Error("Daytona sandbox lease SYNTHETIC is no longer active."))).toBeNull(); + }); + it.each([ "remote_workspace_capture_failed: stage=archive reason=SYNTHETIC/private/path", "remote_workspace_capture_failed: stage=archive exit=128\nSYNTHETIC_PRIVATE", diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index b243095e40d..57a03aafaa4 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -7,6 +7,7 @@ import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/executio import type { RemoteWorkspaceSource } from "@paperclipai/shared"; import { remoteWorkspaceArchiveScript, remoteWorkspaceCaptureFailureMessage, remoteWorkspaceFailureDetail } from "./remote-workspace-snapshot-archive.js"; import type { GitRemoteAuthProvider } from "./git-credentials.js"; +import { classifyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-merge"; export type RemoteWorkspaceTarget = Extract; @@ -31,7 +32,7 @@ const CAPTURE_FAILURE_CHECK = `const captureFailure=${remoteWorkspaceCaptureFail // structured fields from remoteWorkspaceFailureDetail. Anything else is dropped. const SANDBOX_CAPTURE_CAUSE = /^remote_workspace_capture_failed: stage=(?:helper|archive|publish)(?: (?:helper_)?(?:code|exit|signal|syscall|killed|program|name|reason)=[A-Za-z0-9_]{1,64}){0,16}$/; -type RemoteWorkspaceOperation = "prepare" | "repository" | "archive_helper" | "capture" | "restore" | "cleanup"; +type RemoteWorkspaceOperation = "prepare" | "repository" | "archive_helper" | "capture" | "download" | "restore" | "cleanup"; export class RemoteWorkspaceOperationTimeoutError extends Error { constructor(readonly operation: RemoteWorkspaceOperation, readonly timeoutMs: number) { @@ -58,7 +59,9 @@ export class RemoteWorkspaceOperationError extends Error { export function remoteWorkspaceFailureCause(error: unknown): string | null { if (error instanceof RemoteWorkspaceOperationError || error instanceof RemoteWorkspaceOperationTimeoutError) return error.message; if (!(error instanceof Error)) return null; - return remoteWorkspaceCaptureFailureMessage({ message: error.message }) === error.message ? error.message : null; + if (remoteWorkspaceCaptureFailureMessage({ message: error.message }) === error.message) return error.message; + const operation = providerRefusals.get(error); + return operation ? remoteWorkspaceProviderRefusalCause(operation, error) : null; } export const REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE = "remote_workspace_prepare_timeout"; @@ -73,10 +76,56 @@ export class RemoteWorkspacePreparationTimeoutError extends RemoteWorkspaceOpera } } +const TRANSPORT_FAILURE_LABELS: Array<[RegExp, string]> = [ + [/daytona_sandbox_not_found/i, "daytona_sandbox_not_found"], + [/sandbox lease \S+ is no longer active/i, "sandbox_lease_inactive"], + [/\b(?:EACCES|EPERM|permission denied)\b/i, "permission_denied"], + [/\b(?:not (?:started|running)|is stopped|state[^A-Za-z]{1,4}stopped|stopped sandbox)\b/i, "sandbox_not_running"], + [/\b(?:timed? ?out|timeout|ETIMEDOUT)\b/i, "transport_timeout"], + [/\b(?:ECONNRESET|ECONNREFUSED|EPIPE|ENOTFOUND|EAI_AGAIN|socket hang up|fetch failed|channel lost|transport_closed|websocket)\b/i, "transport_closed"], + [/\b(?:401|403|unauthori[sz]ed|forbidden)\b/i, "provider_access_denied"], + [/\b(?:429|rate.?limit(?:ed)?)\b/i, "provider_rate_limited"], + [/\b(?:5\d\d|service unavailable|bad gateway|internal server error)\b/i, "provider_unavailable"], +]; + +// Copy-back errors the provider threw before any exit status, by operation. +// The error itself is rethrown unchanged, so existing callers and classifiers +// keep their contracts; only the recorded cause is derived from this. +const providerRefusals = new WeakMap(); +function noteProviderRefusal(operation: RemoteWorkspaceOperation, error: unknown) { + if (error && typeof error === "object" && !providerRefusals.has(error)) providerRefusals.set(error, operation); +} + +/** + * A copy-back command that never produced an exit status (the provider or its + * transport threw) is named by a closed label and an optional errno-style code. + * Provider text can carry sandbox paths or identifiers, so it is never copied. + */ +function remoteWorkspaceProviderRefusalCause(operation: RemoteWorkspaceOperation, error: unknown): string | null { + // Archive-safety and Git-metadata refusals have their own exact diagnostics. + if (["restore_unsafe_archive", "restore_invalid_metadata"].includes(classifyWorkspaceRestoreFailure(error))) return null; + const texts: string[] = []; + let code: string | null = null; + for (let cause = error, depth = 0; cause && depth < 4; cause = (cause as { cause?: unknown }).cause, depth++) { + if (cause instanceof Error) texts.push(cause.message); + else if (typeof cause === "string") texts.push(cause); + const candidate = (cause as { code?: unknown }).code; + if (!code && typeof candidate === "string" && /^[A-Z][A-Z0-9_]{1,31}$/.test(candidate)) code = candidate; + } + const text = texts.join("\n"); + const label = TRANSPORT_FAILURE_LABELS.find(([pattern]) => pattern.test(text) || (code !== null && pattern.test(code)))?.[1] + ?? "provider_error"; + return `remote_workspace_operation_failed: operation=${operation} transport=${label}${code ? ` code=${code}` : ""}`; +} + async function command(target: RemoteWorkspaceTarget, operation: RemoteWorkspaceOperation, args: string[], env?: Record, timeoutMs = 300_000) { - const result = await target.runner!.execute({ command: "node", args, cwd: target.remoteCwd, + const execute = () => target.runner!.execute({ command: "node", args, cwd: target.remoteCwd, env: { NODE_OPTIONS: "", NODE_PATH: "", ...env }, timeoutMs, bypassSession: true }); + // Copy-back records why the provider refused the command. + const result = operation === "capture" || operation === "archive_helper" + ? await execute().catch((error: unknown) => { noteProviderRefusal(operation, error); throw error; }) + : await execute(); if (result.timedOut) throw new RemoteWorkspaceOperationTimeoutError(operation, timeoutMs); if (result.exitCode !== 0) { const gitOperation = "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace"; @@ -317,7 +366,11 @@ export async function captureRemoteWorkspaceArchive(input: { for await (const chunk of createReadStream(archivePath)) { hash.update(chunk); bytes += chunk.length; } if (hash.digest("hex") !== receipt.sha256 || bytes !== receipt.byteSize) throw new Error("remote_workspace_capture_transfer_mismatch"); return { ...receipt, archivePath, cleanup: () => rm(root, { recursive: true, force: true }) }; - } catch (error) { await rm(root, { recursive: true, force: true }); throw error; } + } catch (error) { + await rm(root, { recursive: true, force: true }); + if (!(error instanceof Error && error.message === "remote_workspace_capture_transfer_mismatch")) noteProviderRefusal("download", error); + throw error; + } } export async function restoreRemoteWorkspaceDirectory(input: { diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index c8da450fbf2..b0fd3e02a67 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -27,7 +27,8 @@ import { canonicalExecutionUuid } from "./execution-identity.js"; import { persistActivity } from "./activity-log.js"; import { captureRemoteWorkspaceArchive, cleanupRemoteWorkspaceArchive, prepareRemoteWorkspaceDirectory, prepareRemoteRepository, remoteWorkspaceRepositoryPath, - REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, RemoteWorkspaceOperationTimeoutError, RemoteWorkspacePreparationTimeoutError, requireRemoteWorkspaceTarget, restoreRemoteWorkspaceDirectory, + REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, RemoteWorkspaceOperationTimeoutError, RemoteWorkspacePreparationTimeoutError, remoteWorkspaceFailureCause, + requireRemoteWorkspaceTarget, restoreRemoteWorkspaceDirectory, } from "./remote-workspace-transport.js"; import { beginWorkspaceSnapshot, getWorkspaceSnapshot, loadWorkspaceSnapshotArchive, uploadWorkspaceSnapshot, @@ -550,8 +551,12 @@ async function retireQuiescedSaves(tx: Transaction, workspace: Workspace, runId: const candidates = await tx.select().from(workspaceOperations).where(and( eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id), eq(workspaceOperations.heartbeatRunId, runId), eq(workspaceOperations.phase, "remote_workspace_snapshot"), - inArray(workspaceOperations.status, ["running", "pending"]), sql`${workspaceOperations.metadata}->>'epoch' = ${String(epoch)}`)).for("update"); + // A save that already ended as failed is retired the same way, so its + // export staging is still cleaned once the final save commits. + inArray(workspaceOperations.status, ["running", "pending", "failed"]), sql`${workspaceOperations.metadata}->>'epoch' = ${String(epoch)}`)).for("update"); for (const candidate of candidates) { + if (candidate.status === "failed" && (object(candidate.metadata?.retiredSave).schema === RETIRED_SAVE_SCHEMA + || !candidate.metadata?.lastFailure)) continue; const work = readSaveWork(candidate.metadata), generation = Number(candidate.metadata?.generation); if (!work || work.unknown || work.activeIds.length || !work.quiescedAt || candidate.metadata?.boundary === "final" || !Number.isSafeInteger(generation) || generation >= final.generation @@ -570,8 +575,20 @@ async function retireQuiescedSaves(tx: Transaction, workspace: Workspace, runId: } } +/** + * The redacted reason a save invocation failed: a server-built remote cause, an + * internal snake_case code, or a fixed fallback. Provider text is never kept. + */ +export function remoteWorkspaceSaveFailure(error: unknown): { code: string; cause?: string } { + const cause = remoteWorkspaceFailureCause(error); + const message = error instanceof Error ? error.message.trim() : ""; + const code = cause ? cause.split(":")[0]! : /^[a-z][a-z0-9_]{2,80}$/.test(message) ? message : "remote_workspace_save_failed"; + return cause ? { code, cause } : { code }; +} + async function settleSaveInvocation(input: { db: Db; workspace: Workspace; runId: string; epoch: number; - location: NonNullable; operationId: string; invocationId: string }) { + location: NonNullable; operationId: string; invocationId: string; + failure?: { code: string; cause?: string } }) { return input.db.transaction(async tx => { const workspace = await currentWorkspace(tx, input.workspace.companyId, input.workspace.id); const [operation] = await tx.select().from(workspaceOperations).where(and(eq(workspaceOperations.id, input.operationId), @@ -581,8 +598,17 @@ async function settleSaveInvocation(input: { db: Db; workspace: Workspace; runId if (!operation || !work || !work.activeIds.includes(input.invocationId)) return false; const activeIds = work.activeIds.filter(id => id !== input.invocationId); if (!activeIds.length && !work.unknown) await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`); + const now = new Date(); + // The last settled invocation of a failed save ends the operation. A later + // retry of the same boundary reopens it; a success still records "succeeded". + const failure = input.failure; + const failed = Boolean(failure) && operation.status === "running" && !activeIds.length && !work.unknown; await tx.update(workspaceOperations).set({ metadata: { ...operation.metadata, - saveWork: { ...work, activeIds, quiescedAt: activeIds.length ? null : new Date().toISOString() } }, updatedAt: new Date() }) + saveWork: { ...work, activeIds, quiescedAt: activeIds.length ? null : now.toISOString() }, + ...(failed && failure ? { lastFailure: { ...failure, at: now.toISOString() } } : {}) }, + ...(failed && failure ? { status: "failed", exitCode: 1, finishedAt: now, + stderrExcerpt: `${[failure.code, failure.cause].filter(Boolean).join("\n")}\n` } : {}), + updatedAt: now }) .where(eq(workspaceOperations.id, operation.id)); await retireQuiescedSaves(tx, workspace, input.runId, input.epoch, input.location); return true; @@ -611,7 +637,9 @@ export async function saveRemoteWorkspace(input: { if (previous.issueId !== issueId) throw new Error("remote_workspace_run_binding_changed"); if (object(previous.metadata?.retiredSave).schema === RETIRED_SAVE_SCHEMA) throw new Error("remote_workspace_snapshot_retired"); const work = readSaveWork(previous.metadata); - const [registered] = await tx.update(workspaceOperations).set({ metadata: { ...previous.metadata, + // A retry of a failed save reopens that boundary's operation. + const reopened = previous.status === "failed" ? { status: "running", finishedAt: null, exitCode: null } : {}; + const [registered] = await tx.update(workspaceOperations).set({ ...reopened, metadata: { ...previous.metadata, saveWork: { schema: SAVE_WORK_SCHEMA, activeIds: [...(work?.activeIds ?? []), invocationId], unknown: work?.unknown ?? true, quiescedAt: null } }, updatedAt: new Date() }).where(eq(workspaceOperations.id, previous.id)).returning(); return registered!; @@ -641,6 +669,7 @@ export async function saveRemoteWorkspace(input: { return created!; }); let captured: Awaited> | null = null; + let failure: { code: string; cause?: string } | undefined; try { const snapshot = await beginWorkspaceSnapshot(input.db, { companyId: input.workspace.companyId, workspaceId: input.workspace.id, operationId: operation.id, runId: input.runId, parentSnapshotId: operation.metadata?.parentSnapshotId as string | null, @@ -688,6 +717,9 @@ export async function saveRemoteWorkspace(input: { eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id))); }); return { snapshotId: saved.id, generation: saved.generation, sha256: saved.sha256, savedAt: at.toISOString() }; + } catch (error) { + failure = remoteWorkspaceSaveFailure(error); + throw error; } finally { // Every upload/capture promise has settled before its durable token clears. // Missing completion evidence remains pinned; cleanup never masks the save error. @@ -695,7 +727,7 @@ export async function saveRemoteWorkspace(input: { catch { logger.warn({ runId: input.runId, operationId: operation.id }, "Remote workspace local archive cleanup failed"); } try { if (await settleSaveInvocation({ db: input.db, workspace: input.workspace, runId: input.runId, epoch: expected.epoch, - location: expected.location, operationId: operation.id, invocationId })) { + location: expected.location, operationId: operation.id, invocationId, failure })) { await cleanupSavedExports({ db: input.db, companyId: input.workspace.companyId, workspaceId: input.workspace.id, location: expected.location, target, assertOwned: input.assertOwned }); }