fix: retry interrupted work-folder uploads with verified fresh streams

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/services/sandbox-work-folders.ts

@@ -7,6 +7,7 @@ import { WORK_FOLDER_SCOPES, type SandboxWorkFolderManifest, type WorkFolderScop
 import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
 import type { StorageProvider } from "../storage/types.js";
 import { loadConfig } from "../config.js";
+import { logger } from "../middleware/logger.js";
 import { createStorageProviderFromConfig } from "../storage/provider-registry.js";
 import { resolveDefaultAgentWorkspaceDir } from "../home-paths.js";
 import { createGitRemoteAuthProvider } from "./git-credentials.js";
@@ -319,7 +320,15 @@ export async function prepareSandboxWorkFolders(input: {
       for (const { binding, root } of bindings) await repositories.checkpoint(binding, root);
       await saveState("saved");
     },
-    async onError() { await saveState("failed", "Files could not be saved; the sandbox must be retained for recovery"); },
+    async onError(error) {
+      const failure = error as { name?: unknown; code?: unknown; $metadata?: { httpStatusCode?: unknown } } | null;
+      // Do not log SDK request objects, headers, file contents or credentials.
+      const label = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9_]{1,80}$/.test(value) ? value : null;
+      logger.warn({ runId: input.runId, errorName: label(failure?.name), errorCode: label(failure?.code),
+        httpStatus: typeof failure?.$metadata?.httpStatusCode === "number" ? failure.$metadata.httpStatusCode : null },
+      "Work folder checkpoint failed; retaining sandbox for recovery");
+      await saveState("failed", "Files could not be saved; the sandbox must be retained for recovery");
+    },
   });
   let completion: Promise<void> | null = null;
   function stop(beforeCompletion?: () => Promise<void>) {

server/src/services/work-folder-repositories.ts

@@ -1,10 +1,10 @@
 import { createHash, randomUUID } from "node:crypto";
-import { Readable, Transform } from "node:stream";
 import { and, eq, inArray, isNull } from "drizzle-orm";
 import { taskRepositoryBindings, workFolderObjects, type Db } from "@paperclipai/db";
 import { validateWorkFilePath } from "@paperclipai/shared";
 import { z } from "zod";
 import { registerWorkFolderObject } from "./work-folder-garbage.js";
+import { uploadWorkFolderObject } from "./work-folder-upload.js";
 import type { StorageProvider } from "../storage/types.js";
 import type { WorkFolderTransport, WorkTreeEntry } from "./work-folder-transport.js";
 
@@ -40,14 +40,9 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr
       const objectKey = entry.kind === "file" && !entry.linkTarget ? `${prefix}blobs/${entry.sha256}` : null;
       if (objectKey && !known.has(objectKey)) await registerWorkFolderObject(db, storage, { objectKey, companyId: binding.companyId, repositoryBindingId: binding.id });
       if (objectKey && !known.has(objectKey) && !(await storage.headObject({ objectKey })).exists) {
-        const hash = createHash("sha256");
-        const verify = new Transform({ transform(chunk: Buffer, _encoding, callback) { hash.update(chunk); callback(null, chunk); },
-          flush(callback) { callback(hash.digest("hex") === entry.sha256 ? undefined : new Error("Repository changed during checkpoint")); } });
-        const source = transport.read(root, entry.path, entry.byteSize);
-        source.on("error", (error) => verify.destroy(error));
-        try {
-          await storage.putObject({ objectKey, body: source.pipe(verify), contentType: "application/octet-stream", contentLength: entry.byteSize });
-        } finally { source.destroy(); verify.destroy(); }
+        await uploadWorkFolderObject(storage, { objectKey, contentType: "application/octet-stream",
+          contentLength: entry.byteSize, sha256: entry.sha256!,
+          createSource: () => transport.read(root, entry.path, entry.byteSize) });
       }
       files.push({ ...entry, objectKey });
     }

server/src/services/work-folder-upload.ts

@@ -0,0 +1,62 @@
+import { createHash } from "node:crypto";
+import { Readable, Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { setTimeout as delay } from "node:timers/promises";
+import type { StorageProvider } from "../storage/types.js";
+
+function transientUploadFailure(error: unknown) {
+  if (!(error instanceof Error)) return false;
+  const failure = error as Error & { code?: string; $metadata?: { httpStatusCode?: number } };
+  // Authentication, validation and ownership failures must never be retried.
+  const status = failure.$metadata?.httpStatusCode;
+  if (status !== undefined) return [408, 429, 500, 502, 503, 504].includes(status);
+  return ["ECONNRESET", "EPIPE", "ETIMEDOUT", "ECONNABORTED", "EAI_AGAIN"].includes(failure.code ?? "");
+}
+
+/** Retry an uncertain object PUT with a fresh, fully verified stream and the same key. */
+export async function uploadWorkFolderObject(storage: Pick<StorageProvider, "putObject">, input: {
+  objectKey: string;
+  contentType: string;
+  contentLength: number;
+  sha256: string;
+  createSource: () => Readable;
+}) {
+  for (let attempt = 0; attempt < 3; attempt++) {
+    const source = input.createSource();
+    const hash = createHash("sha256");
+    let bytes = 0;
+    let validationError: Error | undefined;
+    const changed = () => validationError ??= new Error("Work file changed during upload; retry the checkpoint");
+    const verify = new Transform({
+      transform(chunk: Buffer, _encoding, callback) {
+        bytes += chunk.length;
+        if (bytes > input.contentLength) return callback(changed());
+        hash.update(chunk);
+        callback(null, chunk);
+      },
+      flush(callback) {
+        callback(bytes === input.contentLength && hash.digest("hex") === input.sha256 ? undefined : changed());
+      },
+    });
+    const transferred = pipeline(source, verify);
+    const uploaded = Promise.resolve().then(() => storage.putObject({
+      objectKey: input.objectKey, contentType: input.contentType,
+      contentLength: input.contentLength, body: verify,
+    }));
+    let failure: unknown;
+    try {
+      // Observe both promises immediately. An early HTTP success cannot publish
+      // unvalidated content, and a failed request must release its source reader.
+      await Promise.all([transferred, uploaded]);
+      return;
+    } catch (error) {
+      failure = validationError ?? error;
+    } finally {
+      source.destroy();
+      verify.destroy();
+      await Promise.allSettled([transferred, uploaded]);
+    }
+    if (validationError || attempt === 2 || !transientUploadFailure(failure)) throw failure;
+    await delay(250 * (attempt + 1));
+  }
+}

server/src/services/work-folders.ts

@@ -9,6 +9,7 @@ import { and, asc, eq, gt, isNull, isNotNull, max, sql } from "drizzle-orm";
 import { workFolders, workFiles, workFileOperations, workFolderObjects, type Db } from "@paperclipai/db";
 import { validateWorkFilePath, type WorkFile, type WorkFolderOwner } from "@paperclipai/shared";
 import { registerWorkFolderObject } from "./work-folder-garbage.js";
+import { uploadWorkFolderObject } from "./work-folder-upload.js";
 import type { StorageProvider } from "../storage/types.js";
 import { badRequest, conflict, notFound, payloadTooLarge } from "../errors.js";
 
@@ -92,6 +93,7 @@ export function workFolderService(db: Db, storage: StorageProvider) {
     const spool = path.join(directory, "content");
     const hash = createHash("sha256");
     let byteSize = 0;
+    let sha256: string | null = null;
     let objectKey: string | null = null;
     let discardUpload = false;
     try {
@@ -103,12 +105,13 @@ export function workFolderService(db: Db, storage: StorageProvider) {
           hash.update(chunk);
           callback(null, chunk);
         } }), createWriteStream(spool, { mode: 0o600 }));
+        sha256 = hash.digest("hex");
+        if (input.expectedSha256 && input.expectedSha256 !== sha256) throw conflict("File changed during transfer; retry the checkpoint");
         objectKey = `${folder.companyId}/work-folders/${folder.id}/${randomUUID()}`;
         await registerWorkFolderObject(db, storage, { objectKey, companyId: folder.companyId, folderId: folder.id });
-        await storage.putObject({ objectKey, body: createReadStream(spool), contentLength: byteSize,
-          contentType: input.contentType ?? "application/octet-stream" });
+        await uploadWorkFolderObject(storage, { objectKey, createSource: () => createReadStream(spool),
+          contentLength: byteSize, sha256, contentType: input.contentType ?? "application/octet-stream" });
       }
-      const sha256 = kind === "file" ? hash.digest("hex") : null;
       if (input.expectedSha256 && input.expectedSha256 !== sha256) throw conflict("File changed during transfer; retry the checkpoint");
       const value = { kind, objectKey, byteSize, sha256, executable: input.executable ?? false,
         contentType: input.contentType ?? "application/octet-stream", updatedAt: new Date() };