From e17f3a65cd28cc2f6b550d30db4273503fb962fc Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 27 Sep 2026 16:02:54 -0500 Subject: [PATCH] fix(runtime): stop retrying unsafe native workspace archives Preserve the accepted result and retained sandbox, distinguish repairable unsafe paths from missing sandboxes, and keep transient copy-back failures retryable. Co-Authored-By: Paperclip --- doc/architecture/native-status-arbitration.md | 8 ++- server/src/services/heartbeat.ts | 31 ++++----- .../native-finalization-reconciler.ts | 13 ++-- .../native-run-finalizer-telemetry.test.ts | 41 ++++++++++++ .../native-runtime/native-run-finalizer.ts | 4 +- .../native-workspace-failure.ts | 22 +++++++ ...native-workspace-finalizer-failure.test.ts | 65 +++++++++++++++++++ .../native-workspace-finalizer.ts | 10 +-- 8 files changed, 155 insertions(+), 39 deletions(-) create mode 100644 server/src/services/native-runtime/native-workspace-failure.ts create mode 100644 server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts diff --git a/doc/architecture/native-status-arbitration.md b/doc/architecture/native-status-arbitration.md index 43cc102f19a..292de88458e 100644 --- a/doc/architecture/native-status-arbitration.md +++ b/doc/architecture/native-status-arbitration.md @@ -214,8 +214,12 @@ commit, and retryable or terminal failure. Examples: -- a workspace-finalization failure preserves the claim and records a retryable - error rather than falsely completing the issue; +- a transient workspace-finalization failure preserves the claim and retries + copy-back without submitting another provider turn. Unsafe archive paths or + links stop retries immediately with `native_workspace_sync_out_unsafe_archive`: + the accepted result and sandbox work remain retained, and the issue is blocked + for repair of the link or path before export is retried. A missing sandbox uses + the separate `native_workspace_sync_out_unrecoverable` recovery reason; - a failed provider run preserves partial evidence and schedules recovery; - a status-version race causes bounded reassessment against current issue state; and diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index d20bbd95a27..d256ef114de 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,4 @@ +import { classifyNativeWorkspaceFailure, type NativeWorkspaceFailureCode } from "./native-runtime/native-workspace-failure.js"; import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { externalConversationStateSql, nonIdleSlackIssueCondition } from "./slack-conversation-state.js"; @@ -9323,8 +9324,7 @@ class NativeWorkspaceFinalizeScheduledError extends Error { constructor( readonly original: unknown, readonly terminalFailure: boolean, - readonly reasonCode: - "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable", + readonly reasonCode: NativeWorkspaceFailureCode, ) { super("Native workspace finalization recovery has been scheduled."); this.name = "NativeWorkspaceFinalizeScheduledError"; @@ -24569,31 +24569,20 @@ export function heartbeatService( .limit(1) .then((rows) => rows[0]?.resultId ?? null); if (proposedResult && nativeWorkspaceSync) { - const workspaceFailureMessage = - adapterErr instanceof Error ? adapterErr.message : ""; - const unrecoverable = - workspaceFailureMessage === - "workspace_sync_out_unrecoverable" || - workspaceFailureMessage.includes("daytona_sandbox_not_found"); + const workspaceFailure = classifyNativeWorkspaceFailure(adapterErr); const failure = await recordNativeFinalizationFailure({ db, runId: run.id, - error: new Error( - unrecoverable - ? "native_workspace_sync_out_unrecoverable" - : "native_workspace_sync_out_failed", - ), + error: new Error(workspaceFailure.failureCode), projectRunStatus: true, failureScope: "workspace", - permanent: unrecoverable, + permanent: workspaceFailure.permanent, }); nativeWorkspaceFinalizeScheduled = true; throw new NativeWorkspaceFinalizeScheduledError( adapterErr, failure.phase === "terminal_failure", - unrecoverable - ? "workspace_sync_out_unrecoverable" - : "workspace_sync_out_failed", + workspaceFailure.code, ); } try { @@ -25423,7 +25412,9 @@ export function heartbeatService( stream: "system", level: err.terminalFailure ? "error" : "warn", message: err.terminalFailure - ? "native result is durable, but the sandbox containing unexported workspace changes is unrecoverable" + ? err.reasonCode === "workspace_sync_out_unsafe_archive" + ? "native result is durable; workspace copy-back requires repair of an unsafe link or path in the retained sandbox" + : "native result is durable, but the sandbox containing unexported workspace changes is unrecoverable" : "native result is durable; workspace copy-back will retry without another provider turn", payload: { attempt: coordinator?.attempt ?? null, @@ -25435,13 +25426,13 @@ export function heartbeatService( if (err.terminalFailure) { // The durable coordinator already failed the run, blocked the // issue, and cleared its execution lock. Let ordinary teardown - // release the now-useless lease and return the agent to service. + // release the lease while retaining the sandbox and its unexported work. nativeWorkspaceFinalizeScheduled = false; providerResourceDispositionForRun = "stop_and_retain"; await finalizeAgentStatus( run.agentId, "failed", - "native_workspace_sync_out_unrecoverable", + `native_${err.reasonCode}`, { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) }, ).catch(() => undefined); } diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index c88479166bb..6ad2cebfd64 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -33,6 +33,7 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { issueService } from "../issues.js"; import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; import { reportRunFailure } from "../run-failure-report.js"; +import { classifyNativeWorkspaceFailure } from "./native-workspace-failure.js"; import { resumeNativeWorkspaceFinalization } from "./native-workspace-finalizer.js"; import { dismissObsoleteNativePolicyReviews } from "./obsolete-policy-reviews.js"; import { @@ -858,20 +859,14 @@ export async function reconcileNativeFinalizations( const workspaceFinalizeStatus = operation.status === "succeeded" ? "succeeded" : "failed"; if (workspaceFinalizeStatus === "failed") { - const unrecoverable = operation.stderrExcerpt?.includes( - "workspace_sync_out_unrecoverable", - ); + const workspaceFailure = classifyNativeWorkspaceFailure(new Error(operation.stderrExcerpt ?? "")); const failure = await recordNativeFinalizationFailure({ db, runId: row.runId, - error: new Error( - unrecoverable - ? "native_workspace_sync_out_unrecoverable" - : "native_workspace_sync_out_failed", - ), + error: new Error(workspaceFailure.failureCode), projectRunStatus: true, failureScope: "workspace", - permanent: unrecoverable, + permanent: workspaceFailure.permanent, }); results.push({ ...failure, diff --git a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts index aeaa9f0f352..721c61546e7 100644 --- a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts +++ b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts @@ -1,3 +1,4 @@ +import { classifyNativeWorkspaceFailure } from "./native-workspace-failure.js"; import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; @@ -626,6 +627,46 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a ).resolves.toBe("blocked"); }); + it("retains the accepted result and stops unsafe-archive retries after the first copy-back failure", async () => { + const fixture = await seedNativeRun(); + const resultId = randomUUID(); + const resultJson = { evidence: "accepted provider work" }; + await db.insert(nativeRunResults).values({ + id: resultId, companyId, issueId: fixture.issueId, runId: fixture.runId, + completionContractId: fixture.contractId, serverFingerprint: `fp-${resultId}`, + schemaStatus: "accepted", resultJson, canonicalSha256: `sha-${resultId}`, + }); + await db.update(heartbeatRuns).set({ resultJson: { prpRunTerminalState: "succeeded" } }) + .where(eq(heartbeatRuns.id, fixture.runId)); + await db.insert(nativeRunFinalizations).values({ + runId: fixture.runId, companyId, issueId: fixture.issueId, + phase: "workspace_finalizing", attempt: 1, resultId, + }); + const workspaceFailure = classifyNativeWorkspaceFailure(new Error( + "Daytona syncOut refusing tarball link whose target escapes the extraction dir: .tools/pnpm -> /usr/bin/pnpm", + )); + const failure = await recordNativeFinalizationFailure({ + db, runId: fixture.runId, error: new Error(workspaceFailure.failureCode), + projectRunStatus: true, failureScope: "workspace", permanent: workspaceFailure.permanent, + }); + expect(failure).toMatchObject({ + phase: "terminal_failure", failureCode: "native_workspace_sync_out_unsafe_archive", + nextAttemptAt: null, attempt: 1, + }); + const coordinator = await db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, fixture.runId)).then((rows) => rows[0]!); + expect(coordinator.resultId).toBe(resultId); + expect(coordinator.failureDetail).toMatchObject({ + workspaceFinalizeAttempt: 1, + recoveryOwner: { kind: "board" }, + nextAction: "Repair the unsafe link or path in the retained sandbox, then retry workspace export without submitting another provider turn.", + }); + const run = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)).then((rows) => rows[0]!); + expect(run.resultJson).toMatchObject({ prpRunTerminalState: "succeeded" }); + expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, resultId)).then((rows) => rows[0]?.resultJson)).toEqual(resultJson); + expect(await db.select().from(issues).where(eq(issues.id, fixture.issueId)).then((rows) => rows[0]?.status)).toBe("blocked"); + }); + it("emits exactly one event for a cancel_continuations write (trap 2: :685/:518 overlap)", async () => { const fixture = await seedNativeRun(); // Build the minimal real rows commitNativeStatusDecision's foreign keys diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index afef715688b..6f4d572c029 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -537,7 +537,9 @@ export async function recordNativeFinalizationFailure(input: { nextAction: input.failureScope === "workspace" ? input.permanent - ? "Restore the exact sandbox containing the unexported workspace changes, or resolve the task manually from durable evidence." + ? failureCode === "native_workspace_sync_out_unsafe_archive" + ? "Repair the unsafe link or path in the retained sandbox, then retry workspace export without submitting another provider turn." + : "Restore the exact sandbox containing the unexported workspace changes, or resolve the task manually from durable evidence." : "Retry workspace export and merge from the retained sandbox; do not submit another provider turn." : "Repair the persisted native result or contract discriminator, then resume finalization from the coordinator.", projectRunStatus: input.projectRunStatus, diff --git a/server/src/services/native-runtime/native-workspace-failure.ts b/server/src/services/native-runtime/native-workspace-failure.ts new file mode 100644 index 00000000000..36eade3f3be --- /dev/null +++ b/server/src/services/native-runtime/native-workspace-failure.ts @@ -0,0 +1,22 @@ +import { classifyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-merge"; + +export type NativeWorkspaceFailureCode = + | "workspace_sync_out_failed" + | "workspace_sync_out_unrecoverable" + | "workspace_sync_out_unsafe_archive"; + +/** Keep initial and recovered copy-back failures on the same retry policy. */ +export function classifyNativeWorkspaceFailure(error: unknown): { + code: NativeWorkspaceFailureCode; + failureCode: `native_${NativeWorkspaceFailureCode}`; + permanent: boolean; +} { + const message = error instanceof Error ? error.message.trim() : ""; + const code: NativeWorkspaceFailureCode = + message === "workspace_sync_out_unrecoverable" || message.includes("daytona_sandbox_not_found") + ? "workspace_sync_out_unrecoverable" + : message === "workspace_sync_out_unsafe_archive" || classifyWorkspaceRestoreFailure(error) === "restore_unsafe_archive" + ? "workspace_sync_out_unsafe_archive" + : "workspace_sync_out_failed"; + return { code, failureCode: `native_${code}`, permanent: code !== "workspace_sync_out_failed" }; +} diff --git a/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts b/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts new file mode 100644 index 00000000000..bc95509e8f2 --- /dev/null +++ b/server/src/services/native-runtime/native-workspace-finalizer-failure.test.ts @@ -0,0 +1,65 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { Db } from "@paperclipai/db"; + +const sync = vi.hoisted(() => ({ resume: vi.fn() })); +vi.mock("./native-workspace-sync.js", () => ({ + readNativeWorkspaceSyncReference: () => ({ leaseId: "lease", providerLeaseId: "sandbox", workspaceId: "workspace" }), + resumeNativeWorkspaceSync: sync.resume, +})); +vi.mock("../environments.js", () => ({ environmentService: () => ({ + getLeaseById: async () => ({ id: "lease", companyId: "company", environmentId: "environment", providerLeaseId: "sandbox", status: "active" }), + getById: async () => ({ id: "environment", companyId: "company" }), +}) })); +vi.mock("../environment-execution-target.js", () => ({ resolveEnvironmentExecutionTarget: async () => ({ kind: "remote" }) })); +vi.mock("../workspace-operations.js", () => ({ workspaceOperationService: () => ({ + createRecorder: () => ({ recordOperation: async (input: { run: () => Promise }) => input.run() }), +}) })); +import { classifyNativeWorkspaceFailure } from "./native-workspace-failure.js"; +import { resumeNativeWorkspaceFinalization } from "./native-workspace-finalizer.js"; + +function fixtureDb(): Db { + const responses = [[{ + companyId: "company", runtimeMode: "native", issueId: "issue", resultId: "result", + runnerProfileJson: { nativeWorkspaceSync: {}, nativeExecutionInput: { binding: {} } }, + }], []]; + return { select: () => { + const query = { + from: () => query, innerJoin: () => query, where: () => query, orderBy: () => query, + limit: async () => responses.shift() ?? [], + }; + return query; + } } as unknown as Db; +} + +describe("native workspace finalization failure classification", () => { + beforeEach(() => sync.resume.mockReset()); + it.each([ + ["Daytona syncOut refusing tarball link whose target escapes the extraction dir: .worktrees/task/.tools/pnpm -> ../../../../../../usr/share/nodejs/corepack/dist/pnpm.js", "workspace_sync_out_unsafe_archive"], + ["Daytona syncOut refusing tarball member that escapes the extraction dir: ../private", "workspace_sync_out_unsafe_archive"], + ["daytona_sandbox_not_found", "workspace_sync_out_unrecoverable"], + ["Daytona syncOut directory download failed: timeout", "workspace_sync_out_failed"], + ])("retains a stable code for %s", async (message, expectedCode) => { + sync.resume.mockRejectedValueOnce(new Error(message)); + const result = await resumeNativeWorkspaceFinalization({ db: fixtureDb(), runId: "run", environmentRuntime: {} as never }); + expect(result).toMatchObject({ + status: "failed", exitCode: 1, + stderr: `${expectedCode}\n`, metadata: { workspaceSync: { code: expectedCode } }, + }); + expect(sync.resume).toHaveBeenCalledOnce(); + expect(JSON.stringify(result)).not.toContain(".tools/pnpm"); + }); +}); + + +describe("native workspace retry policy", () => { + it.each([ + [new Error("workspace_sync_out_unsafe_archive\n"), "workspace_sync_out_unsafe_archive", true], + [Object.assign(new Error("redacted"), { code: "WORKSPACE_RESTORE_UNSAFE_ARCHIVE" }), "workspace_sync_out_unsafe_archive", true], + [new Error("Daytona outbound symlink-escape guard command failed (exit 44)"), "workspace_sync_out_unsafe_archive", true], + [new Error("Daytona outbound symlink-escape guard command failed (exit 1)"), "workspace_sync_out_failed", false], + [new Error("archive network download failed"), "workspace_sync_out_failed", false], + [new Error("workspace_sync_out_unrecoverable\n"), "workspace_sync_out_unrecoverable", true], + ])("classifies %s without leaking details", (error, code, permanent) => { + expect(classifyNativeWorkspaceFailure(error)).toEqual({ code, failureCode: `native_${code}`, permanent }); + }); +}); diff --git a/server/src/services/native-runtime/native-workspace-finalizer.ts b/server/src/services/native-runtime/native-workspace-finalizer.ts index 5dbf526f6aa..1e0c71bc9c4 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.ts @@ -8,6 +8,7 @@ import { nativeRunFinalizations, workspaceOperations, } from "@paperclipai/db"; +import { classifyNativeWorkspaceFailure, type NativeWorkspaceFailureCode } from "./native-workspace-failure.js"; import { workspaceOperationService } from "../workspace-operations.js"; import { inspectManagedGitWorktreeBranch } from "../workspace-runtime.js"; import { environmentService } from "../environments.js"; @@ -29,7 +30,7 @@ function readString(value: unknown) { } function workspaceSyncFailure( - code: "workspace_sync_out_failed" | "workspace_sync_out_unrecoverable", + code: NativeWorkspaceFailureCode, ) { return { status: "failed" as const, @@ -189,12 +190,7 @@ export async function resumeNativeWorkspaceFinalization(input: { }, }; } catch (error) { - const code = - error instanceof Error && - (error.message === "workspace_sync_out_unrecoverable" || - error.message.includes("daytona_sandbox_not_found")) - ? "workspace_sync_out_unrecoverable" - : "workspace_sync_out_failed"; + const { code } = classifyNativeWorkspaceFailure(error); return workspaceSyncFailure(code); } }