From e14ca5f3e920963a5bd177e856eb448d7e2f03c2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 1 Nov 2026 07:06:12 -0600 Subject: [PATCH] fix(server): recover remote agent files on an empty controller Co-Authored-By: Paperclip --- doc/DEVELOPING.md | 10 ++- .../2026-10-31-zero-downtime-web-deploys.md | 27 +++++- .../agent-directory-working-copies.test.ts | 76 ++++++++++++++++- .../agent-directory-working-copies.ts | 83 ++++++++++++++++--- .../agent-instruction-working-copies.ts | 2 +- 5 files changed, 183 insertions(+), 15 deletions(-) diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 9f9a070363c..3e23f56507a 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -4181,7 +4181,15 @@ Runner and remote workspace storage. A fresh run still has its initial `legacy` label at this phase; that label is not the selected execution mode. The runtime selector checks the resolved native warm lifecycle before admission and rejects legacy or per-turn execution. External conversation providers keep their existing -exemption because they own no task filesystem. +exemption because they own no task filesystem. Source resolution checks this +contract again before a configured host folder can fall back to local storage. + +A replacement controller reconstructs an active remote agent-directory cache +from the registered sandbox, including unsaved files. The remote directory, +lease and save baseline remain unchanged; current canonical files are never +uploaded over the retained execution. Remote collection and cleanup serialize +through Postgres across controller homes. A missing remote collector retains a +pending save instead of treating an empty local cache as the agent's files. ## Cloud startup configuration diff --git a/doc/plans/2026-10-31-zero-downtime-web-deploys.md b/doc/plans/2026-10-31-zero-downtime-web-deploys.md index 6ee184aca91..ddf3a2d67df 100644 --- a/doc/plans/2026-10-31-zero-downtime-web-deploys.md +++ b/doc/plans/2026-10-31-zero-downtime-web-deploys.md @@ -1,7 +1,30 @@ # Zero-downtime web deployments -Date: 2026-10-31. Status: runtime and Cloud overlap policy merged; qualification -and existing-volume enrollment remain in progress. No hosted acceptance claim. +Date: 2026-10-31. Status (2026-11-01 13:04 UTC): existing-volume staging enrollment +is complete and its read/save/file-integrity checks pass. Active-agent recovery +and the ten-deploy staging campaign remain unqualified. No production canary yet. + +Latest critical path: Core #18543 includes the runtime admission correction and +owned immutable-cache retirement. Review exposed a resolved host-source fallback; +its fix at `11d6eeefbb` passes all 122 workspace-preparation tests. The live test at +`a18f6b85a3` gracefully stopped A and deleted its entire home in 50.4 seconds, then +failed because B treated A's saved agent-directory cache path as execution +identity. The remote execution did not meet the 420-second reattachment deadline. +That failure remains failed evidence; all three owned Daytona sandboxes were +subsequently confirmed absent and the temporary credential revoked. Recovery now +needs a local projection rebuilt from the exact remote lease with its original +save baseline preserved. Focused different-home/same-path/foreign-lease cases pass; +full affected regressions and another live attempt are next. + +Cloud #1102 merged as `461c4aea28` and is serving on the staging controller. +The original enrollment attempt completed using its original qualified image, +a retained detached volume, and the migrated decision key. Its normal app boot +became healthy; the post-enrollment probe passed 20 reads and three saves with no +HTTP/WebSocket failures and verified all 34 files. The original browser tab +reconnected and saved a new task title; lazy Agents navigation also succeeded. +This is migrated-data/app continuity evidence, not a qualifying overlap or active +agent result. Private receipts remain under the task's staging enrollment and +hosted probe directories; no credential or live target identity belongs here. ## Outcome and endpoint diff --git a/server/src/__tests__/agent-directory-working-copies.test.ts b/server/src/__tests__/agent-directory-working-copies.test.ts index 183769a38d6..0fd6a900ea0 100644 --- a/server/src/__tests__/agent-directory-working-copies.test.ts +++ b/server/src/__tests__/agent-directory-working-copies.test.ts @@ -8,13 +8,14 @@ import fs from "node:fs/promises"; import { execFile as execFileCallback } from "node:child_process"; import { promisify } from "node:util"; import * as executionTargetTools from "@paperclipai/adapter-utils/execution-target"; +import * as checkpoints from "../services/agent-file-checkpoints.js"; import * as ssh from "@paperclipai/adapter-utils/ssh"; const execFile = promisify(execFileCallback); import os from "node:os"; import path from "node:path"; import { createHash, randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { agentFileStore, fileHash, inspectAgentFile, snapshotAgentFiles, MAX_AGENT_FILE_BYTES, MAX_AGENT_DIRECTORY_BYTES, MAX_AGENT_DIRECTORY_ENTRIES } from "../services/agent-file-store.js"; import { agents, companies, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -902,6 +903,79 @@ describe("persistent agent directories", () => { expect((await fs.stat(active.localRoot)).isDirectory()).toBe(true); }); + it.each(["new-home", "same-path-empty", "foreign-lease", "other-controller", "missing-collector"] as const)("recovers remote agent files after controller cache loss: %s", async scenario => { + const copy = await run({ warm: true }); + const environmentId = randomUUID(), leaseId = randomUUID(), remoteCwd = "/fixture/retained-task"; + const executionRoot = path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, copy.runId); + await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox" }); + await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: copy.runId, + provider: "daytona", providerLeaseId: "original-sandbox", status: "active", metadata: { remoteCwd } }); + await db.update(agentInstructionWorkingCopies).set({ location: `remote:${environmentId}`, executionRoot, + receipt: { ...copy.receipt, cleanup: { leaseId, remoteCwd } } }).where(eq(agentInstructionWorkingCopies.runId, copy.runId)); + const saved = (await copies.get(companyId, copy.runId))!; + const replacementHome = path.join(home, `replacement-${randomUUID()}`); + const remoteFiles = await fs.mkdtemp(path.join(home, "remote-checkpoint-")); + await fs.mkdir(path.join(remoteFiles, "files", "policy"), { recursive: true }); + await fs.writeFile(path.join(remoteFiles, "files", entryFile), "Remote instructions changed during the turn"); + await fs.writeFile(path.join(remoteFiles, "files", "unsaved.txt"), "Only the running sandbox has this"); + await fs.rm(path.dirname(copy.localRoot), { recursive: true, force: true }); + const capture = vi.spyOn(checkpoints, "captureAgentFileCheckpoint").mockResolvedValue({ directory: remoteFiles, + manifest: { version: 1, entries: [] }, stats: { hashedBytes: 0, copiedBytes: 0, copiedFiles: 0, scannedEntries: 0 }, cleanup: async () => {} }); + const transport = vi.spyOn(executionTargetTools, "prepareAdapterExecutionTargetRuntime").mockImplementation(async options => ({ + target: options.target!, workspaceRemoteDir: options.workspaceRemoteDir!, runtimeRootDir: options.runtimeRootDir!, + assetDirs: {}, additionalSourceDirs: {}, additionalSourceFailures: [], workspaceSyncSnapshot: null, restoreWorkspace: async () => {}, + })); + const ephemeralBefore = process.env.PAPERCLIP_EPHEMERAL_SERVER; + process.env.PAPERCLIP_EPHEMERAL_SERVER = "true"; + if (scenario === "new-home") process.env.PAPERCLIP_HOME = replacementHome; + const remote = { kind: "remote" as const, transport: "sandbox" as const, environmentId, + leaseId: scenario === "foreign-lease" ? randomUUID() : leaseId, remoteCwd, runner: { execute: vi.fn() } }; + try { + if (scenario === "missing-collector") { + const retained = await copies.collectStopped({ companyId, runId: copy.runId }); + expect(retained).toMatchObject({ state: "pending_collection", errorCode: "INSTRUCTION_REMOTE_COLLECTOR_UNAVAILABLE", baseHash: saved.baseHash }); + expect(retained?.receipt?.baseline).toEqual(saved.receipt?.baseline); + expect(retained?.nextAttemptAt).toBeInstanceOf(Date); + expect(capture).not.toHaveBeenCalled(); expect(transport).not.toHaveBeenCalled(); + return; + } + if (scenario === "other-controller") { + await db.transaction(async tx => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`agent-directory:${companyId}:${agentId}`}, 0))`); + await expect(copies.prepare({ ...target(), runId: copy.runId, cwd: remoteCwd, target: remote, warm: true })) + .rejects.toThrow("Agent directory operation is already active"); + expect(capture).not.toHaveBeenCalled(); expect(transport).not.toHaveBeenCalled(); + }); + return; + } + const preparing = copies.prepare({ ...target(), runId: copy.runId, cwd: remoteCwd, target: remote, warm: true }); + if (scenario === "foreign-lease") { + await expect(preparing).rejects.toThrow("Agent directory recovery lease changed"); + expect(capture).not.toHaveBeenCalled(); expect(transport).not.toHaveBeenCalled(); + return; + } + const recovered = (await preparing)!; + expect(recovered.executionRoot).toBe(executionRoot); + expect(recovered.baseHash).toBe(saved.baseHash); + expect(recovered.receipt).toEqual(saved.receipt); + if (scenario === "new-home") { + expect(recovered.localRoot.startsWith(replacementHome)).toBe(true); + await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" }); + } else expect(recovered.localRoot).toBe(copy.localRoot); + expect(await fs.readFile(path.join(recovered.localRoot, "unsaved.txt"), "utf8")).toBe("Only the running sandbox has this"); + expect(await fs.readFile(path.join(recovered.localRoot, entryFile), "utf8")).toBe("Remote instructions changed during the turn"); + expect(capture).toHaveBeenCalledWith(expect.objectContaining({ executionRoot, previous: { version: 1, entries: [] }, target: remote })); + expect(transport).toHaveBeenCalledWith(expect.objectContaining({ workspaceLocalDir: recovered.localRoot, workspaceInboundMode: "adopt_remote" })); + await copies.prepare({ ...target(), runId: copy.runId, cwd: remoteCwd, target: remote, warm: true }); + expect(capture).toHaveBeenCalledTimes(1); + } finally { + process.env.PAPERCLIP_HOME = home; + if (ephemeralBefore === undefined) delete process.env.PAPERCLIP_EPHEMERAL_SERVER; + else process.env.PAPERCLIP_EPHEMERAL_SERVER = ephemeralBefore; + capture.mockRestore(); transport.mockRestore(); + } + }); + it("retains remote cleanup across restart and failed retries until the original lease is cleaned", async () => { const copy = await run(); const environmentId = randomUUID(), leaseId = randomUUID(), remoteCwd = "/fixture/task"; diff --git a/server/src/services/agent-directory-working-copies.ts b/server/src/services/agent-directory-working-copies.ts index 435d295a442..f65b666d0c4 100644 --- a/server/src/services/agent-directory-working-copies.ts +++ b/server/src/services/agent-directory-working-copies.ts @@ -3,7 +3,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { isDeepStrictEqual } from "node:util"; import { and, eq, inArray, isNull, sql } from "drizzle-orm"; -import { agents, environmentLeases, environments, agentInstructionWorkingCopies as copies, type Db } from "@paperclipai/db"; +import { agents, environmentLeases, environments, agentInstructionWorkingCopies as copies, withDedicatedDbConnection, type Db } from "@paperclipai/db"; import { syncDirectoryToSsh, restoreWorkspaceFromSshExecution } from "@paperclipai/adapter-utils/ssh"; import { prepareAdapterExecutionTargetRuntime, runAdapterExecutionTargetShellCommand, type AdapterExecutionTarget, type PreparedAdapterExecutionTargetRuntime } from "@paperclipai/adapter-utils/execution-target"; import { withDirectoryMergeLock, directorySnapshotSha256, parseDirectorySnapshot, serializeDirectorySnapshot, type DirectoryMergeLockOperation } from "@paperclipai/adapter-utils/workspace-restore-merge"; @@ -18,6 +18,7 @@ import type { Environment, EnvironmentLease } from "@paperclipai/shared"; import { agentDirectoryBaselineDigest, agentDirectoryProbeProgram, observeLocalAgentDirectory } from "./agent-directory-probe.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { cachedAgentFileManifest, captureAgentFileCheckpoint, checkpointBaseline, checkpointSnapshot, type AgentFileManifest } from "./agent-file-checkpoints.js"; +import { withWorkspaceLockKeepalive } from "./native-runtime/native-workspace-lock-keepalive.js"; import { logger } from "../middleware/logger.js"; type Copy = typeof copies.$inferSelect; @@ -54,7 +55,16 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string }); } const ownerFile = (row: Copy) => path.join(path.dirname(row.localRoot), "owner.json"); + const disposableRemoteCopy = (row: Copy) => process.env.PAPERCLIP_EPHEMERAL_SERVER === "true" && row.location.startsWith("remote:"); + function projectionRoot(row: Copy): string { + const origin = String(row.receipt?.materializationRunId ?? row.receipt?.directoryRunId ?? row.runId); + if (!/^[a-zA-Z0-9_-]{1,128}$/.test(origin)) throw conflict("Invalid agent directory owner"); + return path.join(path.dirname(resolveManagedInstructionsRoot({ id: row.agentId, companyId: row.companyId, name: "", adapterConfig: {} })), "file-sync", "runs", origin, "live"); + } async function owns(row: Copy) { + // Remote ownership is durable. A marker on a discarded controller cannot + // establish ownership of the retained sandbox on its replacement. + if (disposableRemoteCopy(row)) return row.state !== "superseded" && !row.receipt?.retainedByRunId; if (row.receipt?.warm !== true) return true; return await fs.readFile(ownerFile(row), "utf8").then(s => JSON.parse(s).runId === row.runId).catch(error => error.code === "ENOENT" && row.processStoppedAt !== null); } @@ -159,7 +169,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string ? path.posix.join(input.target.remoteCwd, ".paperclip-runtime", "agent-files", input.agentId, origin) : localRoot; const location = input.target?.kind === "remote" ? `remote:${input.target.environmentId ?? ""}` : "local"; - if (row && (row.localRoot !== localRoot || row.executionRoot !== executionRoot || row.agentId !== input.agentId || row.location !== location)) throw conflict("Agent directory belongs to a different execution environment"); + if (row && ((!disposableRemoteCopy(row) && row.localRoot !== localRoot) || row.executionRoot !== executionRoot || row.agentId !== input.agentId || row.location !== location)) throw conflict("Agent directory belongs to a different execution environment"); if (row && !serialHeld) { // Restart preparation can replace a completed copy under the same run ID. // Join cleanup's lock before resetting its receipt or creating new bytes. @@ -168,7 +178,10 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string return serial(row, () => prepareCopy(input, true), "agent_directory_prepare"); } if (row && !completed.has(row.state) && row.state !== "preparing") { - if (input.target?.kind === "remote" && !transports.has(key(row))) transports.set(key(row), await transport(row, input.target, true)); + if (input.target?.kind === "remote") { + if (disposableRemoteCopy(row)) await restoreProjection(row, input.target); + if (!transports.has(key(row))) transports.set(key(row), await transport(row, input.target, true)); + } return row; } // Register ownership before creating any bytes, so a crash during the copy @@ -241,6 +254,32 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string } } + async function restoreProjection(row: Copy, target: AdapterExecutionTarget) { + const exists = await fs.lstat(row.localRoot).then(stat => { + if (!stat.isDirectory() || stat.isSymbolicLink()) throw conflict("Agent directory projection is not a directory"); + return true; + }, error => { + if (error.code === "ENOENT") return false; + throw error; + }); + if (exists) return; + const lease = await ownedRemoteLease(row); + const cleanup = row.receipt?.cleanup as { leaseId?: string; remoteCwd?: string } | undefined; + if (target.kind !== "remote" || !lease || target.leaseId !== lease.id || lease.status !== "active" + || lease.releasedAt || hasRemoteTerminationReceipt(lease) || cleanup?.remoteCwd !== target.remoteCwd + || row.location !== `remote:${target.environmentId}`) throw conflict("Agent directory recovery lease changed"); + // Fetch a full validated projection. The original save baseline remains in + // Postgres; neither current canonical files nor an empty cache are uploaded. + const captured = await captureAgentFileCheckpoint({ runId: row.runId, localRoot: row.localRoot, + executionRoot: row.executionRoot, previous: { version: 1, entries: [] }, target }); + try { + await fs.cp(path.join(captured.directory, "files"), row.localRoot, { recursive: true }); + if (row.receipt?.warm === true) await fs.writeFile(ownerFile(row), JSON.stringify({ runId: row.runId })); + } catch (error) { + await fs.rm(row.localRoot, { recursive: true, force: true }); + throw error; + } finally { await captured.cleanup(); } + } async function retrieve(row: Copy, target?: AdapterExecutionTarget | null) { if (row.location !== "local") { let runtime = transports.get(key(row)); @@ -356,6 +395,14 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string } async function checkpoint(row: Copy, target?: AdapterExecutionTarget | null, stopped = false): Promise { if (!await owns(row) || row.state === "superseded") return row; + target ??= transports.get(key(row))?.target; + if (row.location !== "local" && target?.kind !== "remote") { + // A missing controller cache is not the remote directory. Retain the + // pending save until its exact remote collector or destruction proof exists. + return patch(row, { ...(stopped ? { state: "pending_collection", processStoppedAt: row.processStoppedAt ?? new Date() } : {}), + errorCode: "INSTRUCTION_REMOTE_COLLECTOR_UNAVAILABLE", errorMessage: "The remote agent directory is retained; its collector is unavailable. No save or discard is claimed.", + nextAttemptAt: stopped ? new Date(Date.now() + 30_000) : null }); + } let failure: unknown; for (let attempt = 0; attempt < 3; attempt++) { let captured: Awaited> | undefined; @@ -575,13 +622,29 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string return result.exitCode === 0 && !result.timedOut; } async function serial(row: Copy, fn: (current: Copy) => Promise, operation: DirectoryMergeLockOperation): Promise { - // Duplicate stop callbacks and restart recovery must not race while moving - // the stopped working copy. This lock is outside the writable tree. - return withDirectoryMergeLock(path.resolve(row.localRoot, "../../.."), async () => { - const current = await get(row.companyId, row.runId); - if (!current) throw notFound("Agent directory copy not found"); - return fn(current); - }, process.env, operation); + const run = async () => { + const lockRoot = path.resolve(disposableRemoteCopy(row) ? projectionRoot(row) : row.localRoot, "../../.."); + if (disposableRemoteCopy(row)) await fs.mkdir(lockRoot, { recursive: true, mode: 0o700 }); + return withDirectoryMergeLock(lockRoot, async () => { + let current = await get(row.companyId, row.runId); + if (!current) throw notFound("Agent directory copy not found"); + if (disposableRemoteCopy(current) && current.localRoot !== projectionRoot(current)) { + current = await patch(current, { localRoot: projectionRoot(current) }); + if (current.localRoot !== projectionRoot(current)) throw conflict("Agent directory projection changed"); + transports.delete(key(current)); + } + return fn(current); + }, process.env, operation); + }; + if (!disposableRemoteCopy(row)) return run(); + // Different controllers have different cache roots. Serialize their active + // collection/cleanup operations in Postgres, without holding row locks over I/O. + return withDedicatedDbConnection(db, dedicated => dedicated.transaction(async tx => { + const [lock] = await tx.execute(sql`select pg_try_advisory_xact_lock(hashtextextended(${`agent-directory:${row.companyId}:${row.agentId}`}, 0)) as acquired, + (select setting::bigint from pg_settings where name = 'idle_in_transaction_session_timeout') as idle_timeout_ms`); + if (lock?.acquired !== true) throw conflict("Agent directory operation is already active"); + return withWorkspaceLockKeepalive(tx as unknown as Db, Number(lock.idle_timeout_ms), run); + })); } return { prepare: (input: Parameters[0]) => prepareCopy(input), hasChanges, adopt, canReuse, recoverUnavailable, recoverStoppedRemote: (row: Copy) => serial(row, current => current.receipt?.retainedByRunId ? Promise.resolve(current) : recoverStoppedRemote(current), "agent_directory_collect"), diff --git a/server/src/services/agent-instruction-working-copies.ts b/server/src/services/agent-instruction-working-copies.ts index 3e7a6363ecf..0d990331d4a 100644 --- a/server/src/services/agent-instruction-working-copies.ts +++ b/server/src/services/agent-instruction-working-copies.ts @@ -121,7 +121,7 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen if (signalsCleanup) await notifyDirectoryCleanup(tx, { companyId: row.companyId, runId: row.runId }); if (signalsCollection) await notifyInstructionCollection(tx, { companyId: row.companyId, runId: row.runId }); const [updated] = await tx.update(copies).set({ ...values, updatedAt: new Date() }).where(and( - scope(row.companyId, row.runId), eq(copies.state, row.state), eq(copies.attempts, row.attempts), eq(copies.baseHash, row.baseHash), + scope(row.companyId, row.runId), eq(copies.state, row.state), eq(copies.attempts, row.attempts), eq(copies.baseHash, row.baseHash), eq(copies.localRoot, row.localRoot), sql`${copies.receipt}->>'retainedByRunId' IS NOT DISTINCT FROM ${typeof row.receipt?.retainedByRunId === "string" ? row.receipt.retainedByRunId : null}`, requireDue ? or(isNull(copies.nextAttemptAt), lte(copies.nextAttemptAt, new Date())) : undefined, condition, )).returning();