Preserve unbound legacy workspaces and contain failed final saves

Bundle the task disposition helper with the installed Paperclip skill. Retry transient read-only file transfers within a fixed deadline and retain sandbox RPC caller provenance for staging failure diagnosis.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/services/heartbeat.ts

@@ -3,7 +3,7 @@ import { githubBrokerEnvironment } from "@paperclipai/adapter-utils/github-launc
 import { cleanupGitHubOperationLaunchers, prepareGitHubOperationLaunchers, startAdapterExecutionTargetPaperclipBridge } from "@paperclipai/adapter-utils/execution-target";
 import fs from "node:fs/promises";
 import { retainUnsavedWorkFolderLease, workFolderSandboxKey } from "./work-folder-retention.js";
-import { hasLegacySandboxWorkspace } from "./legacy-sandbox-workspace.js";
+import { findUnboundLegacyTaskWorkspace, hasLegacySandboxWorkspace } from "./legacy-sandbox-workspace.js";
 import { prepareSandboxWorkFolders } from "./sandbox-work-folders.js";
 import { bindWarmSandboxWorkspace } from "./sandbox-workspace-binding.js";
 import path from "node:path";
@@ -18646,34 +18646,6 @@ export function heartbeatService(
           : null;
       const persistedNativeExecutionWorkspaceId =
         persistedNativeExecutionInput?.binding.executionWorkspaceId ?? null;
-      const requestedExecutionWorkspaceId =
-        persistedNativeExecutionWorkspaceId ??
-        readNonEmptyString(issueRef?.executionWorkspaceId);
-      const existingExecutionWorkspace = requestedExecutionWorkspaceId
-        ? await executionWorkspacesSvc.getById(requestedExecutionWorkspaceId)
-        : null;
-      const nativeRecoveryExecutionWorkspaceId =
-        resolveNativeRecoveryExecutionWorkspaceBinding({
-          bindingId: persistedNativeExecutionWorkspaceId,
-          persistedWorkspaceFound: existingExecutionWorkspace !== null,
-        });
-      const workspaceReuseRequest =
-        resolveExecutionWorkspaceReuseRequestForIssue({
-          issueExecutionWorkspaceId: requestedExecutionWorkspaceId,
-          issueExecutionWorkspacePreference: nativeRecoveryExecutionWorkspaceId
-            ? "reuse_existing"
-            : (issueRef?.executionWorkspacePreference ?? null),
-          existingExecutionWorkspaceStatus:
-            existingExecutionWorkspace?.status ?? null,
-        });
-      const requestedShouldReuseExisting =
-        workspaceReuseRequest.requestedShouldReuseExisting;
-      const reusableExistingExecutionWorkspace =
-        workspaceReuseRequest.existingExecutionWorkspaceAvailable
-          ? existingExecutionWorkspace
-          : null;
-      const requestedReusableExecutionWorkspaceConfig =
-        reusableExistingExecutionWorkspace?.config ?? null;
       const localEnvironment = await environmentsSvc.ensureLocalEnvironment(
         agent.companyId,
       );
@@ -18786,6 +18758,42 @@ export function heartbeatService(
           : selectedEnvironmentId
             ? await environmentsSvc.getById(selectedEnvironmentId)
             : null;
+      const unboundLegacyWorkspaceId = persistedNativeExecutionWorkspaceId ? null
+        : await findUnboundLegacyTaskWorkspace(db, {
+            companyId: agent.companyId, issueId, projectId: issueRef?.projectId ?? null,
+            agentId: agent.id, responsibleUserId: run.responsibleUserId,
+            adapterType: agent.adapterType, environment: selectedEnvironmentForConfig,
+            executionWorkspaceId: readNonEmptyString(issueRef?.executionWorkspaceId),
+            executionWorkspacePreference: issueRef?.executionWorkspacePreference ?? null,
+          });
+      const requestedExecutionWorkspaceId =
+        persistedNativeExecutionWorkspaceId ??
+        readNonEmptyString(issueRef?.executionWorkspaceId) ?? unboundLegacyWorkspaceId;
+      const existingExecutionWorkspace = requestedExecutionWorkspaceId
+        ? await executionWorkspacesSvc.getById(requestedExecutionWorkspaceId)
+        : null;
+      const nativeRecoveryExecutionWorkspaceId =
+        resolveNativeRecoveryExecutionWorkspaceBinding({
+          bindingId: persistedNativeExecutionWorkspaceId,
+          persistedWorkspaceFound: existingExecutionWorkspace !== null,
+        });
+      const workspaceReuseRequest =
+        resolveExecutionWorkspaceReuseRequestForIssue({
+          issueExecutionWorkspaceId: requestedExecutionWorkspaceId,
+          issueExecutionWorkspacePreference: nativeRecoveryExecutionWorkspaceId || unboundLegacyWorkspaceId
+            ? "reuse_existing"
+            : (issueRef?.executionWorkspacePreference ?? null),
+          existingExecutionWorkspaceStatus:
+            existingExecutionWorkspace?.status ?? null,
+        });
+      const requestedShouldReuseExisting =
+        workspaceReuseRequest.requestedShouldReuseExisting;
+      const reusableExistingExecutionWorkspace =
+        workspaceReuseRequest.existingExecutionWorkspaceAvailable
+          ? existingExecutionWorkspace
+          : null;
+      const requestedReusableExecutionWorkspaceConfig =
+        reusableExistingExecutionWorkspace?.config ?? null;
       const sharedWorkspaceConcurrency = resolveSharedWorkspaceConcurrency({
         projectPolicy: projectExecutionWorkspacePolicy,
         issueSettings: issueExecutionWorkspaceSettings,

server/src/services/legacy-sandbox-workspace.ts

@@ -1,6 +1,6 @@
-import { and, eq, isNotNull, isNull, notExists, or, sql } from "drizzle-orm";
-import { environmentLeases, heartbeatRuns, workFolderRuns, type Db } from "@paperclipai/db";
-import type { EnvironmentLease } from "@paperclipai/shared";
+import { and, desc, eq, inArray, isNotNull, isNull, notExists, or, sql } from "drizzle-orm";
+import { environmentLeases, executionWorkspaces, heartbeatRuns, workFolderRuns, type Db } from "@paperclipai/db";
+import type { Environment, EnvironmentLease } from "@paperclipai/shared";
 
 function record(value: unknown): Record<string, unknown> | null {
   return value !== null && typeof value === "object" && !Array.isArray(value)
@@ -12,6 +12,46 @@ export function hasLegacySandboxWorkspace(lease: Pick<EnvironmentLease, "metadat
     || record(lease.metadata?.reusableSandboxLease)?.version === 1;
 }
 
+/** Older releases retained sandboxes without binding their workspace to the task. */
+export async function findUnboundLegacyTaskWorkspace(db: Db, input: {
+  companyId: string; issueId: string | null; projectId: string | null;
+  agentId: string; responsibleUserId: string | null; adapterType: string;
+  executionWorkspaceId: string | null; executionWorkspacePreference: string | null;
+  environment: Pick<Environment, "id" | "driver" | "config"> | null;
+}) {
+  if (!input.issueId || !input.projectId || input.executionWorkspaceId || input.executionWorkspacePreference
+    || input.environment?.driver !== "sandbox" || input.environment.config.reuseLease !== true) return null;
+  const [candidate] = await db.select({ lease: environmentLeases }).from(environmentLeases)
+    .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, environmentLeases.heartbeatRunId),
+      eq(heartbeatRuns.companyId, environmentLeases.companyId)))
+    .innerJoin(executionWorkspaces, and(eq(executionWorkspaces.id, environmentLeases.executionWorkspaceId),
+      eq(executionWorkspaces.companyId, environmentLeases.companyId)))
+    .where(and(eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.issueId, input.issueId),
+      eq(environmentLeases.environmentId, input.environment.id),
+      eq(environmentLeases.leasePolicy, "reuse_by_environment"),
+      inArray(environmentLeases.status, ["retained", "released"]),
+      isNotNull(environmentLeases.providerLeaseId),
+      sql`${environmentLeases.metadata}->>'driver' = 'sandbox'`,
+      sql`${environmentLeases.metadata}->>'workFolderLayout' is distinct from 'scoped'`,
+      sql`${environmentLeases.metadata}->'reusableSandboxLease'->>'version' = '1'`,
+      eq(heartbeatRuns.agentId, input.agentId),
+      input.responsibleUserId === null ? isNull(heartbeatRuns.responsibleUserId)
+        : eq(heartbeatRuns.responsibleUserId, input.responsibleUserId),
+      eq(executionWorkspaces.projectId, input.projectId), eq(executionWorkspaces.status, "active"),
+      or(isNull(executionWorkspaces.sourceIssueId), eq(executionWorkspaces.sourceIssueId, input.issueId)),
+      notExists(db.select({ id: workFolderRuns.runId }).from(workFolderRuns).where(and(
+        eq(workFolderRuns.companyId, input.companyId), sql`${workFolderRuns.manifest}->>'taskId' = ${input.issueId}`)))))
+    .orderBy(desc(environmentLeases.createdAt), desc(environmentLeases.id)).limit(1);
+  if (!candidate) return null;
+  const lease = await bindLegacySandboxIdentity(db, candidate.lease as EnvironmentLease);
+  const scope = record(lease.metadata?.reusableSandboxLease);
+  if (scope?.version !== 2 || scope.issueId !== input.issueId || scope.responsibleUserId !== input.responsibleUserId
+    || scope.adapterType !== input.adapterType || scope.environmentId !== input.environment.id
+    || scope.executionWorkspaceId !== lease.executionWorkspaceId) return null;
+  // Normal workspace freshness and provider sentinel checks still run before use.
+  return lease.executionWorkspaceId;
+}
+
 /** Keep the old sync/restore contract even after its provider sandbox expires. */
 export async function taskUsesLegacySandboxWorkspace(db: Db, companyId: string, issueId: string | null) {
   if (!issueId) return false;

server/src/services/plugin-worker-manager.ts

@@ -3292,6 +3292,10 @@ export function createPluginWorkerHandle(
     timeoutMs?: number,
     executeLogSink?: ExecuteLogSink,
   ): Promise<HostToWorkerMethods[M][1]> {
+    // A worker response arrives on a different stack from its caller. Preserve
+    // command-call provenance so late sandbox failures identify their owner
+    // without recording command arguments, credentials, or worker payloads.
+    const caller = method === "environmentExecute" ? new Error("Sandbox command requested here") : undefined;
     const rpcPromise = new Promise<HostToWorkerMethods[M][1]>((resolve, reject) => {
       if (!childProcess?.stdin?.writable) {
         reject(
@@ -3347,7 +3351,9 @@ export function createPluginWorkerHandle(
           if (isJsonRpcSuccessResponse(response)) {
             settle(resolve, response.result as HostToWorkerMethods[M][1]);
           } else if ("error" in response && response.error) {
-            settle(reject, new JsonRpcCallError(response.error));
+            const error = new JsonRpcCallError(response.error);
+            if (caller) error.cause = caller;
+            settle(reject, error);
           } else {
             settle(reject, new Error(`Unexpected response format for "${method}"`));
           }

server/src/services/work-folder-checkpointer.ts

@@ -6,6 +6,7 @@ export function startWorkFolderCheckpointer(input: {
   onError(error: unknown): Promise<void>;
 }) {
   let active: Promise<void> | null = null;
+  let finalFlush: Promise<void> | null = null;
   let stopped = false;
   function checkpoint() {
     const pending = (async () => {
@@ -33,6 +34,13 @@ export function startWorkFolderCheckpointer(input: {
   }
   return {
     flush,
-    async stop() { stopped = true; clearInterval(timer); await flush(); },
+    stop() {
+      stopped = true;
+      clearInterval(timer);
+      // Success and error teardown can both call stop. Never begin another
+      // save after the caller has already terminalized this run on failure.
+      finalFlush ??= flush();
+      return finalFlush;
+    },
   };
 }

server/src/services/work-folder-transport.ts

@@ -1,6 +1,7 @@
 import { readFile } from "node:fs/promises";
 import { Readable } from "node:stream";
 import { randomUUID } from "node:crypto";
+import { setTimeout as delay } from "node:timers/promises";
 import path from "node:path";
 import { z } from "zod";
 import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
@@ -15,11 +16,33 @@ let source: Promise<string> | undefined;
 // Linux's 128 KiB single-argument limit, including a provider shell wrapper.
 const WRITE_CHUNK_BYTES = 48 * 1024;
 
+function transientReadFailure(error: unknown) {
+  if (!(error instanceof Error)) return false;
+  const code = (error as Error & { code?: string }).code;
+  return ["ECONNRESET", "EPIPE", "EAI_AGAIN", "ECONNABORTED"].includes(code ?? "")
+    || error.message === "socket hang up";
+}
+
 export function workFolderTransport(runner: CommandManagedRuntimeRunner) {
   async function command(input: Record<string, unknown>): Promise<unknown> {
     source ??= readFile(new URL("./scripts/work-folder-io.mjs", import.meta.url), "utf8");
-    const result = await runner.execute({ command: "node", args: ["--input-type=module", "-e", await source,
-      Buffer.from(JSON.stringify(input)).toString("base64")], bypassSession: true, timeoutMs: 120_000 });
+    const args = ["--input-type=module", "-e", await source, Buffer.from(JSON.stringify(input)).toString("base64")];
+    const readOnly = ["home", "scan", "read"].includes(String(input.operation));
+    const deadline = Date.now() + 120_000;
+    let result;
+    for (let attempt = 0; ; attempt++) {
+      try {
+        result = await runner.execute({ command: "node", args, bypassSession: true,
+          timeoutMs: Math.max(1, deadline - Date.now()) });
+        break;
+      } catch (error) {
+        // A lost read response is safe to repeat. Staging writes, publishes and
+        // moves may already have happened, so never replay them here.
+        const waitMs = 250 * (attempt + 1);
+        if (!readOnly || attempt >= 2 || !transientReadFailure(error) || Date.now() + waitMs >= deadline) throw error;
+        await delay(waitMs);
+      }
+    }
     if (result.exitCode !== 0 || result.timedOut) throw new Error(`Work folder ${String(input.operation)} failed: ${result.stderr.slice(0, 1500)}`);
     return JSON.parse(result.stdout);
   }