diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md index c8f1883d462..29c0d19f613 100644 --- a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -146,6 +146,14 @@ recovery source validation and company boundaries. ## Queued turns during offline enrollment +Offline workspace import preserves a pending repository request when preparation +has never published a target directory, commit, branch or failure. The request +stays pending in the database. It is excluded from the saved snapshot's materialized +repository descriptors, so the next authorized sandbox admission can acquire it +normally. Existing targets, preparation in progress, repair states, unreadable +paths and live imports remain blocked. Publication rechecks the request and +materialized source before transferring file authority. + The task drain stops new executions. It does not consume queued turns. After Cloud has stopped every application deployment, offline maintenance preserves never-started queued run and wake rows unchanged. Workspace, instruction-file, diff --git a/server/src/services/remote-workspace-migration.test.ts b/server/src/services/remote-workspace-migration.test.ts index 9da4182d8f1..4054a99e23d 100644 --- a/server/src/services/remote-workspace-migration.test.ts +++ b/server/src/services/remote-workspace-migration.test.ts @@ -9,7 +9,7 @@ import { pathToFileURL } from "node:url"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { and, eq } from "drizzle-orm"; import { activityLog, agents, environmentLeases, heartbeatRuns, nativeRunFinalizations, nativeRunResults, completionContracts, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots, - executionWorkspaceRuntimeLeases, executionWorkspaces, issueRecoveryActions, issues, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices } from "@paperclipai/db"; + executionWorkspaceRepositories, executionWorkspaceRuntimeLeases, executionWorkspaces, issueRecoveryActions, issues, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; import { createStorageService } from "../storage/service.js"; @@ -73,6 +73,72 @@ describe.skipIf(!support.supported)("remote workspace migration", () => { return { companyId, workspaceId, issueId: issue!.id, cwd }; } async function workspace(id: string) { return (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, id)))[0]!; } + async function pendingRepository(f: Awaited>) { + const [repository] = await db.insert(executionWorkspaceRepositories).values({ companyId: f.companyId, + executionWorkspaceId: f.workspaceId, requestedByIssueId: f.issueId, repositoryIdentity: "https://example.invalid/pending.git", + repoUrl: "https://example.invalid/pending.git", relativePath: ".paperclip-repositories/pending", requestedRef: "HEAD", + requestKey: "preserve-pending", requestKeys: ["preserve-pending"] }).returning(); + return repository!; + } + it("preserves an unmaterialized pending repository request during offline migration", async () => { + const f = await seed(), repository = await pendingRepository(f); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + await migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true }); + const authority = readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)!; + expect(authority.source).toEqual({ kind: "empty" }); + expect(await db.select().from(executionWorkspaceRepositories).where(eq(executionWorkspaceRepositories.id, repository.id))) + .toEqual([repository]); + await fs.rm(f.cwd, { recursive: true }); + const reference = await getWorkspaceSnapshot(db, { ...f, snapshotId: authority.snapshotId! }); + const archivePath = path.join(root, "pending.pcws"), targetDir = path.join(root, "restored-pending"); + await loadWorkspaceSnapshotArchive({ storage, reference: reference!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: reference!.manifestDigest }); + expect(await fs.readFile(path.join(targetDir, "saved.txt"), "utf8")).toBe("original saved work\n"); + expect(await fs.lstat(path.join(targetDir, repository.relativePath)).catch(() => null)).toBeNull(); + }); + it("rejects publication if a pending repository becomes ready during capture", async () => { + const f = await seed(), repository = await pendingRepository(f); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + const putFile = storage.putFile.bind(storage); + vi.spyOn(storage, "putFile").mockImplementationOnce(async input => { + const result = await putFile(input); + await db.update(executionWorkspaceRepositories).set({ state: "ready", pinnedCommit: "b".repeat(40) }) + .where(eq(executionWorkspaceRepositories.id, repository.id)); + return result; + }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true })) + .rejects.toThrow("remote_workspace_migration_source_changed"); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n"); + }); + it.each(["online", "preparing", "needs_repair", "pinned", "branched", "failure", "directory", "symlink", "parent_symlink", "parent_file", "escaping_path"] as const)( + "keeps an unresolved repository request blocked when %s", async state => { + const f = await seed(), repository = await pendingRepository(f); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + if (state === "preparing" || state === "needs_repair") await db.update(executionWorkspaceRepositories) + .set({ state }).where(eq(executionWorkspaceRepositories.id, repository.id)); + if (state === "pinned") await db.update(executionWorkspaceRepositories).set({ pinnedCommit: "a".repeat(40) }) + .where(eq(executionWorkspaceRepositories.id, repository.id)); + if (state === "branched") await db.update(executionWorkspaceRepositories).set({ branchName: "saved" }) + .where(eq(executionWorkspaceRepositories.id, repository.id)); + if (state === "failure") await db.update(executionWorkspaceRepositories).set({ failureCode: "repository_preparation_failed" }) + .where(eq(executionWorkspaceRepositories.id, repository.id)); + if (state === "escaping_path") await db.update(executionWorkspaceRepositories).set({ relativePath: "../absent" }) + .where(eq(executionWorkspaceRepositories.id, repository.id)); + if (state === "parent_symlink") await fs.symlink("missing-parent", path.dirname(path.join(f.cwd, repository.relativePath))); + if (state === "parent_file") await fs.writeFile(path.dirname(path.join(f.cwd, repository.relativePath)), "preserve parent file\n"); + if (state === "directory" || state === "symlink") { + const target = path.join(f.cwd, repository.relativePath); await fs.mkdir(path.dirname(target), { recursive: true }); + if (state === "directory") { await fs.mkdir(target); await fs.writeFile(path.join(target, "partial.txt"), "keep partial work\n"); } + else await fs.symlink("missing-target", target); + } + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: state !== "online" })) + .rejects.toThrow("repository_preparation_unresolved"); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n"); + if (state === "directory") expect(await fs.readFile(path.join(f.cwd, repository.relativePath, "partial.txt"), "utf8")) + .toBe("keep partial work\n"); + }); async function nestedFixture() { const parent = await seed(), workspaceId = randomUUID(); const cwd = path.join(parent.cwd, ".paperclip", "worktrees", "child"); diff --git a/server/src/services/remote-workspace-migration.ts b/server/src/services/remote-workspace-migration.ts index bfc28c4a809..d5b63d8de58 100644 --- a/server/src/services/remote-workspace-migration.ts +++ b/server/src/services/remote-workspace-migration.ts @@ -53,6 +53,34 @@ const gitSourceProof = (source: ArchivedGitMigrationSource | null) => source ? { ...(source.retainedBranch ? { retainedBranch: source.retainedBranch } : {}), } : null; +async function isUnmaterializedRepositoryRequest(workspace: Workspace, + repository: typeof executionWorkspaceRepositories.$inferSelect) { + if (!workspace.cwd || repository.state !== "pending" || repository.pinnedCommit + || repository.branchName || repository.failureCode) return false; + const relative = repository.relativePath; + if (!relative || relative.includes("\\") || path.posix.isAbsolute(relative) + || path.posix.normalize(relative) !== relative || relative.split("/").some(part => !part || part === "." || part === "..")) return false; + const root = await fs.lstat(workspace.cwd).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (!root?.isDirectory() || root.isSymbolicLink()) return false; + let parent = workspace.cwd; + for (const part of relative.split("/").slice(0, -1)) { + parent = path.join(parent, part); + const entry = await fs.lstat(parent).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (!entry) return true; + if (!entry.isDirectory() || entry.isSymbolicLink()) return false; + } + return !await fs.lstat(path.join(workspace.cwd, relative)).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); +} + async function inspect(db: ReadDb, input: { companyId: string; workspaceId: string; operationId?: string; writersStopped?: boolean }) { const [workspace] = await db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))); if (!workspace) throw new Error("remote_workspace_migration_workspace_missing"); @@ -236,10 +264,13 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri legacyRecoverySources.sort((a, b) => a.runId.localeCompare(b.runId)); const repositories = await db.select().from(executionWorkspaceRepositories).where(and(eq(executionWorkspaceRepositories.companyId, workspace.companyId), eq(executionWorkspaceRepositories.executionWorkspaceId, workspace.id))); - if (repositories.some(repository => repository.state !== "ready")) reasons.push("repository_preparation_unresolved"); + for (const repository of repositories.filter(repository => repository.state !== "ready")) { + if (!input.writersStopped || !await isUnmaterializedRepositoryRequest(workspace, repository)) reasons.push("repository_preparation_unresolved"); + } const source: RemoteWorkspaceSource = workspace.repoUrl ? { kind: "git", repoUrl: workspace.repoUrl, ref: archivedGitSource?.commit ?? workspace.baseRef, branchName: workspace.branchName } : { kind: "empty" }; - if (repositories.length) source.repositories = repositories.map(repository => ({ id: repository.id, relativePath: repository.relativePath, + const materialized = repositories.filter(repository => repository.state === "ready"); + if (materialized.length) source.repositories = materialized.map(repository => ({ id: repository.id, relativePath: repository.relativePath, repoUrl: repository.repoUrl, ref: repository.requestedRef, ...(repository.pinnedCommit ? { pinnedCommit: repository.pinnedCommit } : {}) })); return { workspace, reasons: [...new Set(reasons)], authority, marker, source, issueBinding: issue?.executionWorkspaceId ?? null, retainedDirectory, archivedGitSource, legacyRecoverySources, finalizedRecoveryHolds }; } @@ -387,6 +418,7 @@ export async function migrateRemoteWorkspace(input: { eq(environmentLeases.id, allocation.leaseId))).for("update", { noWait: true }); } const validation = await inspect(tx as unknown as Db, { ...input, operationId: operation.id }); + if (!input.source && canonical(validation.source) !== canonical(reserved.source)) throw new Error("remote_workspace_migration_source_changed"); if (validation.issueBinding !== reserved.issueBinding) throw new Error("remote_workspace_migration_source_changed"); if (Boolean(validation.retainedDirectory) !== Boolean(reserved.retainedDirectory)) throw new Error("remote_workspace_migration_source_changed"); if (canonical(gitSourceProof(validation.archivedGitSource)) !== canonical(gitSourceProof(reserved.archivedGitSource))) throw new Error("remote_workspace_migration_source_changed"); diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index e298568efae..7a997f41482 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -4,6 +4,7 @@ import { buildNativeHarnessBackupManifest, verifyNativeHarnessBackup } from "./n import { createHash, randomUUID } from "node:crypto"; import { execFile, spawn } from "node:child_process"; import net from "node:net"; +import { createServer } from "node:http"; import { promisify } from "node:util"; import fs from "node:fs/promises"; import os from "node:os"; @@ -12,7 +13,7 @@ import { pathToFileURL } from "node:url"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { and, eq, sql } from "drizzle-orm"; import { activityLog, agents, agentTaskSessions, agentWakeupRequests, applyPendingMigrations, closeRegisteredClients, companies, createDb, environmentLeases, environments, - executionWorkspaces, executionWorkspaceSnapshots, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issues, livenessDb, nativeRunFinalizations, projects, projectWorkspaces, workspaceOperations } from "@paperclipai/db"; + executionWorkspaceRepositories, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issues, livenessDb, nativeRunFinalizations, projects, projectWorkspaces, workspaceOperations } from "@paperclipai/db"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; import { buildProjectMentionHref, type Environment, type EnvironmentLease } from "@paperclipai/shared"; @@ -38,6 +39,7 @@ import { terminalizeLegacyExecution } from "./legacy-execution-recovery.js"; import { environmentRuntimeService } from "./environment-runtime.js"; import { legacyControllerClaim, renewLegacyControllerLease } from "./legacy-controller-lease.js"; import { migrateRemoteWorkspace } from "./remote-workspace-migration.js"; +import { executionWorkspaceRepositoryService, taskRepositoryRelativePath } from "./execution-workspace-repositories.js"; import { registerRemoteWorkspaceSnapshotRetentionWork, remoteWorkspaceSnapshotRetentionService } from "./remote-workspace-snapshot-retention.js"; import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { issueService } from "./issues.js"; @@ -683,6 +685,76 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem expect(await fs.readFile(path.join(current, "runner", "state.json"), "utf8")).toBe("unchanged opaque provider state"); }, 60_000); + it("cold restores a workspace and prepares its preserved pending repository through normal admission", async () => { + const original = path.join(root, "pending-original"), origin = path.join(root, "pending-origin.git"); + await fs.mkdir(original); await fs.mkdir(origin); + await fs.writeFile(path.join(original, "saved.txt"), "saved before repository request\n"); + const git = (cwd: string, ...args: string[]) => exec("git", ["-C", cwd, "-c", "user.name=Pending fixture", + "-c", "user.email=pending@example.invalid", ...args]); + await git(origin, "init", "--initial-branch=main"); + await fs.writeFile(path.join(origin, "code.txt"), "acquired on next admission\n"); + await git(origin, "add", "."); await git(origin, "commit", "-m", "Pending baseline"); + await git(origin, "update-server-info"); + const server = createServer(async (request, response) => { + const target = path.resolve(root, new URL(request.url!, "http://localhost").pathname.slice(1)); + if (!target.startsWith(`${origin}/.git/`)) { response.writeHead(403).end(); return; } + try { response.end(await fs.readFile(target)); } catch { response.writeHead(404).end(); } + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + try { + const address = server.address(); if (!address || typeof address === "string") throw new Error("test server not bound"); + const repoUrl = "https://github.com/paperclip-fixtures/pending"; + const fixtureOrigin = `http://127.0.0.1:${address.port}/pending-origin.git/.git`, relativePath = taskRepositoryRelativePath(repoUrl); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, ids.run)); + const [local] = await db.insert(executionWorkspaces).values({ companyId: ids.company, sourceIssueId: ids.issue, + mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", name: "Pending request", + cwd: original, providerRef: original }).returning(); + await db.update(issues).set({ executionWorkspaceId: local.id }).where(eq(issues.id, ids.issue)); + const [repository] = await db.insert(executionWorkspaceRepositories).values({ companyId: ids.company, + executionWorkspaceId: local.id, requestedByIssueId: ids.issue, repoUrl, repositoryIdentity: repoUrl, + relativePath, requestedRef: "HEAD", requestKey: "pending-request", requestKeys: ["pending-request"] }).returning(); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + await migrateRemoteWorkspace({ db, storage, companyId: ids.company, workspaceId: local.id, apply: true, writersStopped: true }); + expect(readRemoteWorkspaceAuthority((await workspace(local.id)).metadata)!.source).toEqual({ kind: "empty" }); + expect(await db.select().from(executionWorkspaceRepositories).where(eq(executionWorkspaceRepositories.id, repository.id))) + .toEqual([repository]); + await fs.rm(original, { recursive: true }); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "false"); + const nextRun = randomUUID(), nextLease = randomUUID(); + await db.insert(heartbeatRuns).values({ id: nextRun, companyId: ids.company, agentId: ids.agent, nativeIssueId: ids.issue, + responsibleUserId: "local-board", runtimeMode: "native", status: "running" }); + const [allocated] = await db.insert(environmentLeases).values({ id: nextLease, companyId: ids.company, environmentId: ids.environment, + heartbeatRunId: nextRun, issueId: ids.issue, provider: "daytona", providerLeaseId: `allocation-${nextLease}`, status: "active" }).returning(); + const reserved = await reserveRemoteWorkspace({ db, storage, companyId: ids.company, issueId: ids.issue, runId: nextRun, + agentId: ids.agent, environmentId: ids.environment, projectId: null, projectWorkspaceId: null, + existingWorkspaceId: local.id, source: { kind: "empty" } }); + const remote = await target("pending-restored-sandbox"); + const execute = remote.runner!.execute; + remote.runner!.execute = async input => { + const raw = input.args?.at(-1); + if (typeof raw === "string" && raw.startsWith("{")) { + const payload = JSON.parse(raw) as { repoUrl?: string; authArgs?: string[] }; + if (payload.repoUrl === repoUrl && Array.isArray(payload.authArgs)) input = { ...input, + args: [...input.args!.slice(0, -1), JSON.stringify({ ...payload, + authArgs: [...payload.authArgs, "-c", `url.${fixtureOrigin}.insteadOf=${repoUrl}`] })] }; + } + return execute(input); + }; + const restored = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: nextRun, + lease: allocated as EnvironmentLease, target: remote }); + expect(await fs.readFile(path.join(restored.remoteRoot, "saved.txt"), "utf8")).toBe("saved before repository request\n"); + const prepared = await executionWorkspaceRepositoryService(db).prepareForAdmission({ companyId: ids.company, + issueId: ids.issue, workspaceId: local.id, cwd: restored.remoteRoot, agentId: ids.agent, runId: nextRun, + responsibleUserId: "local-board", remoteTarget: remote, remoteLease: restored.lease }); + expect(prepared).toMatchObject([{ id: repository.id, relativePath, + pinnedCommit: (await git(origin, "rev-parse", "HEAD")).stdout.trim() }]); + expect(await fs.readFile(path.join(restored.remoteRoot, relativePath, "code.txt"), "utf8")).toBe("acquired on next admission\n"); + expect(await db.select().from(executionWorkspaceRepositories).where(eq(executionWorkspaceRepositories.id, repository.id))) + .toMatchObject([{ ...repository, state: "ready", pinnedCommit: prepared[0]!.pinnedCommit, + branchName: prepared[0]!.branchName, updatedAt: expect.any(Date) }]); + } finally { await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); } + }, 60_000); + it.each(["path", "file-url"] as const)("cold restores an imported secondary %s repository and acquires only a new descriptor", async sourceKind => { const origin = path.join(root, "import-origin"), original = path.join(root, "old-checkout"); await fs.mkdir(origin); await fs.mkdir(original);