diff --git a/doc/plans/2026-10-31-zero-downtime-web-deploys.md b/doc/plans/2026-10-31-zero-downtime-web-deploys.md index ddf3a2d67df..27c3be61b2d 100644 --- a/doc/plans/2026-10-31-zero-downtime-web-deploys.md +++ b/doc/plans/2026-10-31-zero-downtime-web-deploys.md @@ -1,30 +1,45 @@ # Zero-downtime web deployments -Date: 2026-10-31. Status (2026-11-01 13:04 UTC): existing-volume staging enrollment -is complete and its read/save/file-integrity checks pass. Active-agent recovery -and the ten-deploy staging campaign remain unqualified. No production canary yet. +Date: 2026-10-31. Status (2026-11-01 13:28 UTC): existing-volume staging enrollment +is complete and its read/save/file-integrity checks pass. Active-agent recovery, +idle behavior and the ten-deploy staging campaign remain unqualified. No production +canary yet. Latest critical path: Core #18543 includes the runtime admission correction and owned immutable-cache retirement. Review exposed a resolved host-source fallback; -its fix at `11d6eeefbb` passes all 122 workspace-preparation tests. The live test at -`a18f6b85a3` gracefully stopped A and deleted its entire home in 50.4 seconds, then -failed because B treated A's saved agent-directory cache path as execution -identity. The remote execution did not meet the 420-second reattachment deadline. -That failure remains failed evidence; all three owned Daytona sandboxes were -subsequently confirmed absent and the temporary credential revoked. Recovery now -needs a local projection rebuilt from the exact remote lease with its original -save baseline preserved. Focused different-home/same-path/foreign-lease cases pass; -full affected regressions and another live attempt are next. +its fix passes all 122 workspace-preparation tests. The live test at `e14ca5f3e9` +now reconstructs the agent-file projection, deletes A's entire home after graceful +shutdown (22.8 seconds), and transfers ownership of the same remote execution. +It then fails because adoption tries to rematerialize launch skills from A's +deleted controller directory. That failure remains failed evidence; all three +owned Daytona sandboxes were subsequently confirmed absent and the temporary +credential revoked. Adoption now skips preparation of launch assets for the +already-running provider. The regression fails before the fix and passes after +it; all 71 related adoption/recovery transport cases pass with real local Runner +processes and simulated providers. No paid-model success is inferred. + +Review also identified save quotas incorrectly applied during cache recovery. +Recovery now uses the existing inbound remote projection and preserves path/link +validation without enforcing save quotas on admission. The over-quota case still +rejects an ordinary save. A broad directory run passed 110 cases but hit three +cleanup timing failures; all 20 cleanup/cache-loss cases passed in isolation with +the same timeouts. Server and Runner TypeScript checks pass. A rebuilt live retry +and exact-head PR review are next. Cloud #1102 merged as `461c4aea28` and is serving on the staging controller. -The original enrollment attempt completed using its original qualified image, -a retained detached volume, and the migrated decision key. Its normal app boot -became healthy; the post-enrollment probe passed 20 reads and three saves with no -HTTP/WebSocket failures and verified all 34 files. The original browser tab -reconnected and saved a new task title; lazy Agents navigation also succeeded. -This is migrated-data/app continuity evidence, not a qualifying overlap or active -agent result. Private receipts remain under the task's staging enrollment and -hosted probe directories; no credential or live target identity belongs here. +The original enrollment completed using its original qualified image, a retained +detached volume, and the migrated decision key. The post-enrollment probe passed +20 reads and three saves with no HTTP/WebSocket failures and verified all 34 files. +The original browser tab reconnected and persisted a task title. Server logs later +showed old Agents chunks returned 404 and navigation recovered via a page reload: +this is migration continuity evidence, not uninterrupted old-tab asset loading. + +The first eight-minute natural database-idle observation failed. A bounded SQL +attribution window is underway and is explicitly not an idle observation. Normal +automatic sleep was correctly refused because this development fixture is not +eligible for the free/demo idle-sleep policy; manual sleep/wake is still pending. +Private receipts remain under the task's live-test, staging enrollment and hosted +probe directories; no credential or live target identity belongs here. ## Outcome and endpoint @@ -88,8 +103,8 @@ Measure one-time setup separately from repeated attempts. Keep failed evidence. | --- | --- | --- | | Prior prerequisites | #18403 and all five storage workstreams merged | Retain the ephemeral filesystem contract | | Core implementation | #18518, #18526, #18529, #18530, #18531 and #18533 merged; local packaged migration, browser overlap, cold restoration and durable command proofs recorded below | Complete live remote execution, accounting and later continuation qualification | -| Cloud implementation | #1096, #1098, #1099, #1100, #1101 and #1102 merged | Await automatic staging recovery revision, then verify repeated Railway overlap behavior | -| Staging | Original attempt has migrated data and preserved the signing key; volume detached and retained; failed final boot held in maintenance | Resume the same boot attempt after #1102, then qualify ten app-only replacements and lifecycle | +| Cloud implementation | #1096, #1098, #1099, #1100, #1101 and #1102 merged; #1102 serving on staging | Verify repeated Railway overlap behavior after Core recovery qualification | +| Staging | Original enrollment complete; migrated data and signing key preserved; volume detached and retained | Diagnose idle behavior, verify sleep/wake and repair, then qualify ten app-only replacements | | Production | No canary mutation performed | Proceed only after the staging and active-agent acceptance gates pass | Keep PRs, tested revisions, artifact identities, evidence paths, deployment targets diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index b1749f4653a..8219cc6f600 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -20,7 +20,7 @@ import { } from "node:fs/promises"; import { createHash, randomUUID } from "node:crypto"; import { execFileSync, spawn } from "node:child_process"; -import { readFileSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; import { bindAcpxAgentFiles } from "../drivers/acpx/agent-files-binding.js"; import { resolveRunnerProviderAssetsRoot } from "../drivers/acpx/provider-assets-root.js"; import { createServer } from "node:http"; @@ -7646,9 +7646,14 @@ it("rotates PRP authority in place for a warm cross-run attachment", async () => } }, 30_000); -it("recovers an active second warm business run with its opaque attachment identity", async () => { +it("recovers an active second warm business run after deleting its controller context assets", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-warm-second-active-")); const firstRoot = join(root, "first-controller"), nextRoot = join(root, "next-controller"); + const assetRoot = join(root, "retired-controller-assets"); + const skillRoot = join(assetRoot, "skill"), instructionRoot = join(assetRoot, "instructions"); + await mkdir(skillRoot, { recursive: true }); await mkdir(instructionRoot, { recursive: true }); + await writeFile(join(skillRoot, "SKILL.md"), "# Pinned skill\n"); + await writeFile(join(instructionRoot, "AGENTS.md"), "Pinned instructions\n"); const server = createServer(); let authority: DurablePrpControlPlane | null = null, generation = 0; let checkpoint: durableControlPlane.DurablePrpCheckpoint | undefined; @@ -7670,6 +7675,7 @@ it("recovers an active second warm business run with its opaque attachment ident runId: "run-warm-first-healthy", normalizedSessionId: "session-warm-second-active", turnId: "turn-first", itemId: "item-first" }; const options = { runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, codexArgs: fakeCodexArgs(root, "--emit-tool-call", "--durable-tool-ids"), stateDirectory: firstRoot, + runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), prpIdentity: initialIdentity, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, controlPlaneRegistration: registration }; const first = createCapabilityRunnerdCodexTransport(options); const within = async (label: string, promise: Promise) => { @@ -7707,6 +7713,8 @@ it("recovers an active second warm business run with its opaque attachment ident await activeCore.flushCheckpoint(); await within("controller detach", first.detachControllerForRestart()); expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + await rm(assetRoot, { recursive: true, force: true }); + expect(existsSync(assetRoot)).toBe(false); checkpoint!.restoreSessionDirectory(nextRoot); const noSpawn = vi.fn(() => { throw new Error("Duplicate Runner launch"); }); const openedThread = opened.thread as Record; @@ -7758,6 +7766,7 @@ it("recovers an active second warm business run with its opaque attachment ident if (runnerPid) { try { process.kill(-runnerPid, "SIGKILL"); } catch {} } server.closeAllConnections(); if (server.listening) await new Promise(done => server.close(() => done())); + for (const controller of [firstRoot, nextRoot]) await releaseMaterializedNativeRuntimeSkills(join(controller, "codex-home", "skills")); await rm(root, { recursive: true, force: true }); } }, 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 811133f5b0a..fbf1a9f6c6f 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -6046,16 +6046,16 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#assertStartupOpen(); // Route and immutable endpoint admission precedes every launch-material // write. A refused transition leaves its retained home/context untouched. - if (runtimeContext !== null && !this.options.prepareLaunch && this.#sourceRecoveryIdentity === null) { + if (runtimeContext !== null && preparesProviderLaunch && this.#sourceRecoveryIdentity === null) { writeFileSync( localRuntimeContextPath, `${JSON.stringify(runtimeContext)}\n`, { mode: 0o600 }, ); } - if (provider === "aws_agentcore" && !this.options.prepareLaunch) + if (provider === "aws_agentcore" && preparesProviderLaunch) mkdirSync(codexHome, { recursive: true, mode: 0o700 }); - if (!this.options.prepareLaunch && provider === "codex" && this.options.homeMode !== "preconfigured" && this.#sourceRecoveryIdentity === null) { + if (preparesProviderLaunch && provider === "codex" && this.options.homeMode !== "preconfigured" && this.#sourceRecoveryIdentity === null) { await releaseMaterializedNativeRuntimeSkills( resolve(localCodexHome, "skills"), ); diff --git a/server/src/__tests__/agent-directory-working-copies.test.ts b/server/src/__tests__/agent-directory-working-copies.test.ts index 0fd6a900ea0..fcf3c5ebd6a 100644 --- a/server/src/__tests__/agent-directory-working-copies.test.ts +++ b/server/src/__tests__/agent-directory-working-copies.test.ts @@ -8,7 +8,6 @@ import fs from "node:fs/promises"; import { execFile as execFileCallback } from "node:child_process"; import { promisify } from "node:util"; import * as executionTargetTools from "@paperclipai/adapter-utils/execution-target"; -import * as checkpoints from "../services/agent-file-checkpoints.js"; import * as ssh from "@paperclipai/adapter-utils/ssh"; const execFile = promisify(execFileCallback); import os from "node:os"; @@ -16,7 +15,7 @@ import path from "node:path"; import { createHash, randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq, sql } from "drizzle-orm"; -import { agentFileStore, fileHash, inspectAgentFile, snapshotAgentFiles, MAX_AGENT_FILE_BYTES, MAX_AGENT_DIRECTORY_BYTES, MAX_AGENT_DIRECTORY_ENTRIES } from "../services/agent-file-store.js"; +import { agentFileStore, fileHash, inspectAgentFile, inspectAgentDirectory, snapshotAgentFiles, MAX_AGENT_FILE_BYTES, MAX_AGENT_DIRECTORY_BYTES, MAX_AGENT_DIRECTORY_ENTRIES } from "../services/agent-file-store.js"; import { agents, companies, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { agentInstructionRevisionService } from "../services/agent-instruction-revisions.js"; @@ -903,7 +902,7 @@ describe("persistent agent directories", () => { expect((await fs.stat(active.localRoot)).isDirectory()).toBe(true); }); - it.each(["new-home", "same-path-empty", "foreign-lease", "other-controller", "missing-collector"] as const)("recovers remote agent files after controller cache loss: %s", async scenario => { + it.each(["new-home", "same-path-empty", "foreign-lease", "other-controller", "missing-collector", "over-quota"] as const)("recovers remote agent files after controller cache loss: %s", async scenario => { const copy = await run({ warm: true }); const environmentId = randomUUID(), leaseId = randomUUID(), remoteCwd = "/fixture/retained-task"; const executionRoot = path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, copy.runId); @@ -919,11 +918,14 @@ describe("persistent agent directories", () => { await fs.writeFile(path.join(remoteFiles, "files", entryFile), "Remote instructions changed during the turn"); await fs.writeFile(path.join(remoteFiles, "files", "unsaved.txt"), "Only the running sandbox has this"); await fs.rm(path.dirname(copy.localRoot), { recursive: true, force: true }); - const capture = vi.spyOn(checkpoints, "captureAgentFileCheckpoint").mockResolvedValue({ directory: remoteFiles, - manifest: { version: 1, entries: [] }, stats: { hashedBytes: 0, copiedBytes: 0, copiedFiles: 0, scannedEntries: 0 }, cleanup: async () => {} }); + if (scenario === "over-quota") { + const handle = await fs.open(path.join(remoteFiles, "files", "too-large.bin"), "w"); + try { await handle.truncate(MAX_AGENT_FILE_BYTES + 1); } finally { await handle.close(); } + } const transport = vi.spyOn(executionTargetTools, "prepareAdapterExecutionTargetRuntime").mockImplementation(async options => ({ target: options.target!, workspaceRemoteDir: options.workspaceRemoteDir!, runtimeRootDir: options.runtimeRootDir!, - assetDirs: {}, additionalSourceDirs: {}, additionalSourceFailures: [], workspaceSyncSnapshot: null, restoreWorkspace: async () => {}, + assetDirs: {}, additionalSourceDirs: {}, additionalSourceFailures: [], workspaceSyncSnapshot: null, + restoreWorkspace: async () => { await fs.cp(path.join(remoteFiles, "files"), options.workspaceLocalDir, { recursive: true }); }, })); const ephemeralBefore = process.env.PAPERCLIP_EPHEMERAL_SERVER; process.env.PAPERCLIP_EPHEMERAL_SERVER = "true"; @@ -936,7 +938,7 @@ describe("persistent agent directories", () => { expect(retained).toMatchObject({ state: "pending_collection", errorCode: "INSTRUCTION_REMOTE_COLLECTOR_UNAVAILABLE", baseHash: saved.baseHash }); expect(retained?.receipt?.baseline).toEqual(saved.receipt?.baseline); expect(retained?.nextAttemptAt).toBeInstanceOf(Date); - expect(capture).not.toHaveBeenCalled(); expect(transport).not.toHaveBeenCalled(); + expect(transport).not.toHaveBeenCalled(); return; } if (scenario === "other-controller") { @@ -944,14 +946,14 @@ describe("persistent agent directories", () => { await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`agent-directory:${companyId}:${agentId}`}, 0))`); await expect(copies.prepare({ ...target(), runId: copy.runId, cwd: remoteCwd, target: remote, warm: true })) .rejects.toThrow("Agent directory operation is already active"); - expect(capture).not.toHaveBeenCalled(); expect(transport).not.toHaveBeenCalled(); + expect(transport).not.toHaveBeenCalled(); }); return; } const preparing = copies.prepare({ ...target(), runId: copy.runId, cwd: remoteCwd, target: remote, warm: true }); if (scenario === "foreign-lease") { await expect(preparing).rejects.toThrow("Agent directory recovery lease changed"); - expect(capture).not.toHaveBeenCalled(); expect(transport).not.toHaveBeenCalled(); + expect(transport).not.toHaveBeenCalled(); return; } const recovered = (await preparing)!; @@ -964,15 +966,20 @@ describe("persistent agent directories", () => { } else expect(recovered.localRoot).toBe(copy.localRoot); expect(await fs.readFile(path.join(recovered.localRoot, "unsaved.txt"), "utf8")).toBe("Only the running sandbox has this"); expect(await fs.readFile(path.join(recovered.localRoot, entryFile), "utf8")).toBe("Remote instructions changed during the turn"); - expect(capture).toHaveBeenCalledWith(expect.objectContaining({ executionRoot, previous: { version: 1, entries: [] }, target: remote })); + expect(transport).toHaveBeenNthCalledWith(1, expect.objectContaining({ adapterKey: "agent-files-recovery", workspaceRemoteDir: executionRoot, + workspaceBaseline: { exclude: [".paperclip-runtime"], entries: new Map() }, workspaceInboundMode: "adopt_remote" })); + if (scenario === "over-quota") { + expect((await fs.stat(path.join(recovered.localRoot, "too-large.bin"))).size).toBe(MAX_AGENT_FILE_BYTES + 1); + await expect(inspectAgentDirectory(recovered.localRoot)).rejects.toThrow(); + } expect(transport).toHaveBeenCalledWith(expect.objectContaining({ workspaceLocalDir: recovered.localRoot, workspaceInboundMode: "adopt_remote" })); await copies.prepare({ ...target(), runId: copy.runId, cwd: remoteCwd, target: remote, warm: true }); - expect(capture).toHaveBeenCalledTimes(1); + expect(transport).toHaveBeenCalledTimes(2); } finally { process.env.PAPERCLIP_HOME = home; if (ephemeralBefore === undefined) delete process.env.PAPERCLIP_EPHEMERAL_SERVER; else process.env.PAPERCLIP_EPHEMERAL_SERVER = ephemeralBefore; - capture.mockRestore(); transport.mockRestore(); + transport.mockRestore(); } }); diff --git a/server/src/services/agent-directory-working-copies.ts b/server/src/services/agent-directory-working-copies.ts index f65b666d0c4..c1fe658aab6 100644 --- a/server/src/services/agent-directory-working-copies.ts +++ b/server/src/services/agent-directory-working-copies.ts @@ -268,17 +268,24 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string if (target.kind !== "remote" || !lease || target.leaseId !== lease.id || lease.status !== "active" || lease.releasedAt || hasRemoteTerminationReceipt(lease) || cleanup?.remoteCwd !== target.remoteCwd || row.location !== `remote:${target.environmentId}`) throw conflict("Agent directory recovery lease changed"); - // Fetch a full validated projection. The original save baseline remains in - // Postgres; neither current canonical files nor an empty cache are uploaded. - const captured = await captureAgentFileCheckpoint({ runId: row.runId, localRoot: row.localRoot, - executionRoot: row.executionRoot, previous: { version: 1, entries: [] }, target }); + // Reconstruct a private mirror without applying save quotas to admission. + // An empty merge baseline downloads all remote bytes without uploading any; + // the original save baseline remains unchanged in Postgres. + await fs.mkdir(row.localRoot, { recursive: true, mode: 0o700 }); + let projection: PreparedAdapterExecutionTargetRuntime | undefined; try { - await fs.cp(path.join(captured.directory, "files"), row.localRoot, { recursive: true }); + projection = await prepareAdapterExecutionTargetRuntime({ target, runId: row.runId, adapterKey: "agent-files-recovery", + workspaceLocalDir: row.localRoot, workspaceRemoteDir: row.executionRoot, + runtimeRootDir: sandboxTransferRoot(row, target.remoteCwd), syncWorkspace: true, + workspaceInboundMode: "adopt_remote", workspaceBaseline: { exclude: [".paperclip-runtime"], entries: new Map() }, + workspaceGitSnapshot: null, workspaceRepositories: [], workspaceFileMode: "all", workspaceExclude: [".paperclip-runtime", ".paperclip-runtime/**"] }); + await projection.restoreWorkspace(); + await inspectAgentDirectory(row.localRoot, false); if (row.receipt?.warm === true) await fs.writeFile(ownerFile(row), JSON.stringify({ runId: row.runId })); } catch (error) { await fs.rm(row.localRoot, { recursive: true, force: true }); throw error; - } finally { await captured.cleanup(); } + } finally { await projection?.cleanupWorkspaceSnapshot?.(); } } async function retrieve(row: Copy, target?: AdapterExecutionTarget | null) { if (row.location !== "local") {