diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.md index 4e7b4e352cd..995332f5a28 100644 --- a/doc/native-workspace-finalization-recovery.md +++ b/doc/native-workspace-finalization-recovery.md @@ -1,5 +1,20 @@ # Native workspace finalization ownership and recovery +Online legacy workspace import may preserve an unresolved terminal source while +unrelated object-backed work continues. This is a file-authority handoff. It +does not finish the old result, clear its recovery action, stop a provider, or +start another turn. Keep every descriptor, retained allocation, and finalization +checkpoint for the recovery owner. + +The migration owner must hold the existing per-run physical finalizer advisory +locks across capture and publication. It must reject a non-null coordinator +owner, a durable physical copyback owner, a live controller lease, or an active +adapter control. An expired owner is still an owner. Historical agent PID fields +remain unchanged; they do not certify that the provider stopped. Recheck the +exact source proof under publication row locks. A changed source or checkpoint +keeps local authority. Provider stop verification and any later file export +remain the recovery owner's separate obligations. + Periodic heartbeat recovery runs one pass at a time in each server process. A tick that arrives during a slow pass reports its age and starts no duplicate. A later tick can start a fresh pass after the current work settles. Shutdown diff --git a/doc/plans/2026-11-06-online-ordinary-source-fence.md b/doc/plans/2026-11-06-online-ordinary-source-fence.md new file mode 100644 index 00000000000..89b46e67696 --- /dev/null +++ b/doc/plans/2026-11-06-online-ordinary-source-fence.md @@ -0,0 +1,161 @@ +# Online ordinary workspace source fences + +The importer can preserve an idle legacy source while unrelated object-backed +turns continue. This change does not assert `writersStopped` and does not use the +instance-wide offline admission gate. + +## Ownership + +`workspace-source-lock.ts` retains the existing per-company/task source advisory +key and adds a managed-project key. The project key covers the physical project +namespace, including new issues that have no workspace row yet. Heartbeat source +selection/materialization and workspace lifecycle owners retain the same key. +Configured aliases of canonical isolated task directories retain that directory's +original task key. Cross-company aliases are rejected. + +The filesystem materializer keeps its database-free API. It retains physical +keys when called inside an admission owned by the database service. The runtime +call sites that can write these controller namespaces have these owners: + +| Mutation entry | Source admission | +| --- | --- | +| Heartbeat preparation and its source/repository resolvers | Task keys and dynamically retained physical roots for the full preparation | +| Workspace reopen, expiration and cleanup | Existing task/source and lifecycle gateways | +| Rehome and nested-clone conversion | Shared move gateway retains task and project keys through the move | +| Runtime commands with a workspace record | Workspace lifecycle gateway | +| Project runtime commands without a workspace record | Physical project/task keys before provisioning and command startup | + +Standalone materialization and source-resolution callers retain their existing +filesystem API. The source fence is established at the database-owning mutation +entry rather than inferred as a new requirement of that helper API. + +The importer derives a bounded physical alias closure rather than accepting a +caller-supplied proof. It holds every existing alias's task and lifecycle keys, +plus the project key where applicable. It checks the closure and its ordinary +writer, service, operation and repository blockers again before publication. +Future preparation waits at its first project key; acquiring another physical +project while already holding one fails instead of creating a lock cycle. +Migration admission uses nonblocking lock acquisition. + +The authority update commits on the connection holding the source fence. The +activity publication follows that commit. A dropped fence connection cannot +publish through another pool connection. Preparation also refreshes a selected +local workspace if migration published remote authority while selection waited. + +## Capture and recovery + +The archive owner retains all stable-read, mutation, cancellation, normalization, +restore and digest checks. Scoped ordinary captures reject included Git metadata +before reading it. Bare repository shapes, detached Git administration and +object-store alternates also require the larger Git fence. The self-contained +remote script contract remains intact. +The existing nested imported-child gate remains in force. + +Terminal native recovery sources use the canonical online preservation helper. +The importer holds the finalizer's existing physical advisory key, re-inspects +after acquiring it, and compares journaled fingerprints under publication row +locks. Coordinator ownership must be null, including an expired owner. Provider +allocations, finalization checkpoints and task warnings remain present. Unknown +file recovery and other aliases' unproven recovery remain blockers. + +The finalizer's known failed-export watchdog can retain a prepared source. Its +company, task, run, failure cause and canonical action fingerprint must match. +The coordinator must have a terminal failure with no owner or future retry. +The board-owned action must have no wake policy or separate file-recovery marker. +The proof retains `preparedSource` explicitly and never calls it finalized. The +original action, prepared descriptor and allocations remain unchanged. Its full +action/source fingerprint is checked under the same physical finalizer lock. + +The separate company-scoped `reconcileOnlineWorkspaceOperations` entry retires +only a superseded operation with an exact later successful finalization receipt, +an accepted result and a committed ownerless finalizer. It reuses the canonical +row/advisory locks and writes the audit in the same transaction. It does not run +workspace work, settle recovery or change provider/task authority. The offline +entry retains its global deployment prerequisites. Inventory remains read-only. + +## Measured cohort + +The read-only inventory initially found 45 ordinary aliases blocked only by +physical overlap. A second bounded source probe on +`4d554f08218b04df49ec0f2a0d5c5dfdfaf65660` at +`2026-11-06T22:14:40.198Z` checked the three exact managed `_default` roots: + +| Project | Workspace aliases | Source tasks | Root `.git` | Imported nested children | +| --- | ---: | ---: | --- | ---: | +| Shared root A | 40 | 13 | absent | 0 | +| Shared root B | 3 | 1 | absent | 0 | +| Shared root C | 2 | 1 | absent | 0 | + +These are observational counts, not live migration results. A full included-tree +capture still rejects embedded Git or source changes. There are no live holds, +imports, cancellations or source filesystem writes in this qualification. + +## Verification and remaining work + +The focused fixtures exercise the actual importer, storage/archive restore, +heartbeat preparation, PostgreSQL source/lifecycle/finalizer locks, an unrelated +running remote turn, closure mutation, lost exclusion and retained provider +obligations. Run from `server/`: + +```sh +node ../cli/node_modules/tsx/dist/cli.mjs ../packages/db/src/test-postgres-command.ts -- \ + node ../node_modules/vitest/vitest.mjs run \ + src/services/remote-workspace-online-source.test.ts \ + src/services/isolated-remote-workspace-source-lock.test.ts \ + src/services/remote-workspace-migration.test.ts \ + src/services/remote-workspace-migration-concurrency.test.ts \ + src/services/heartbeat/workspace-preparation.test.ts --fsModuleCache +``` + +The full goal still includes the remaining Git/shared and recovery workspaces. +The next extension must derive the Git administrative/common-object/alternates +physical closure and make actual preparation, reopen and cleanup owners retain +its keys before admitting those sources. Exact immutable nested-child coverage +can then use the existing archive owner under that closure. Git stores are not +assumed immutable, and completed or unreferenced history is not discarded. + +The runtime-control route reacquires the same source/lifecycle fence for the +filesystem restore after it publishes admission. Its visible running operation +continues to block migration between admission and restore. The later service +start retains its existing independent lifecycle fence. + +`reconcileOnlineWorkspaceOperations(db, { companyId, signal })` is an explicit +operator apply entry. The source qualification console can call this owner +with one exact company before online imports. Inventory never invokes it and +never settles operation rows as a side effect. The existing offline maintenance +entry retains its independent all-writers-stopped prerequisites. + +Alias inspection bounds the company cohort to 1,000 records. A separate +minimal-path scan checks foreign physical aliases in pages of 256, up to +10,240 records. This safety bound remains conservative because lexical-prefix +filtering could hide a foreign symlink alias. Foreign rows do not consume the +company cohort bound. An inaccessible path or scan limit leaves the managed +source explicitly unproven. It cannot fall back to a weaker singleton import. +No foreign metadata is returned. + +Runtime service exclusion also uses the physical working directory. An active +project service can have a different project-workspace ID and no execution +workspace ID. The company-scoped service scan checks up to 1,000 unsettled +rows; local path uncertainty blocks the import. Both runtime startup owners +hold the workspace, base and configured service source keys before provision +or persistence. The final rendered service CWD must already be fenced before +workspace link updates or spawn. Runs with no services retain their previous +filesystem-free path. + +A read-only source observation at `2026-11-06T23:20:49.282Z` found 254 pending +legacy records and zero unsettled local-process services. No service handoff was +required by that observation. This is not a coherent or locked migration proof. + +Serial source admission exposed a run-scoped preview reuse collision: two +execution workspaces with one caller invocation ID could reuse one healthy +service after the first start completed. Run-scoped reuse identity now includes +the execution-workspace owner. The public scope ID and explicit project/agent +sharing stay unchanged. The existing exposure-pair owner regression exercises +real listeners, a disposable database and a fake broker. + +Local reopen acquires its declared and physical project source keys before the +lifecycle transaction. It retains the recorded base and project-workspace source +before reconstruction. Nested restore therefore cannot wait for the project +source while holding the lifecycle lock. The actual reopen contention regression +holds the project key on another connection, checks that no lifecycle key is +held, then releases the source and checks the restored nested commit and files. diff --git a/server/src/__tests__/execution-workspace-runtime-control-conflict.test.ts b/server/src/__tests__/execution-workspace-runtime-control-conflict.test.ts index d3837d10fbb..ee560472b95 100644 --- a/server/src/__tests__/execution-workspace-runtime-control-conflict.test.ts +++ b/server/src/__tests__/execution-workspace-runtime-control-conflict.test.ts @@ -402,6 +402,28 @@ describe("execution workspace runtime control conflict and failure reconciliatio expect(mockWorkspaceOperationService.assertRuntimeControlAvailable).not.toHaveBeenCalled(); }); + it("reacquires the lifecycle fence for restore after command admission", async () => { + let held = false, restores = 0; + mockLifecycleLock.mockImplementation(async (_db, _id, run) => { + expect(held).toBe(false); + held = true; + try { return await run({}); } finally { held = false; } + }); + mockEnsurePersistedExecutionWorkspaceAvailable.mockImplementationOnce(async () => { + expect(mockPublishRuntimeOperation).toHaveBeenCalledOnce(); + expect(held).toBe(true); + restores++; + return { cwd: "/tmp/lane-a" }; + }); + mockStartRuntimeServices.mockResolvedValueOnce([]); + const response = await request(await createApp()) + .post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-services/start`).send({}); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(restores).toBe(1); + expect(mockLifecycleLock).toHaveBeenCalledTimes(2); + expect(held).toBe(false); + }); + it("tears down residue and records a stopped desired state when a start fails", async () => { mockStartRuntimeServices.mockRejectedValue( new Error( diff --git a/server/src/__tests__/git-workspace-expiration.test.ts b/server/src/__tests__/git-workspace-expiration.test.ts index c3a9abfbec0..9ab0acd7429 100644 --- a/server/src/__tests__/git-workspace-expiration.test.ts +++ b/server/src/__tests__/git-workspace-expiration.test.ts @@ -19,6 +19,8 @@ import { materializeManagedProjectWorkspace } from "../services/managed-reposito import { materializeIsolatedRemoteWorkspaceSource } from "../services/isolated-remote-workspace-source.js"; import { realizeExecutionWorkspace, ensurePersistedExecutionWorkspaceAvailable } from "../services/workspace-runtime.js"; import { executionWorkspaceService } from "../services/execution-workspaces.js"; +import { withExecutionWorkspaceLifecycleLock } from "../services/execution-workspace-lifecycle-lock.js"; +import { withWorkspaceSourceLocks } from "../services/workspace-source-lock.js"; import { prepareProjectRepositoryWorkspaces, mergeExecutionWorkspaceMetadataForPersistence } from "../services/heartbeat/workspaces.js"; import { describeGitExpirationFailure, expireReconstructableGitWorkspace } from "../services/git-workspace-expiration.js"; import { nestedWorktreeBranchPrefix, restoreNestedRepositoryWorktree, withNestedRepositoryBaseLock } from "../services/nested-repository-worktrees.js"; @@ -184,6 +186,70 @@ describeEmbeddedPostgres("reconstructable Git expiration", () => { sharedBase: resolveManagedProjectWorkspaceDir({ companyId: f.companyId, projectId: f.projectId!, repoName: "nested" }) }; } + it("restores expired managed nested contents under the runtime-control source fence", async () => { + const f = await nestedFixture(true, false); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: true, baseExpired: false }); + const workspace = await f.row(); + let entered!: () => void, release!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + const pending = new Promise(resolve => { release = resolve; }); + const sourceOwner = withWorkspaceSourceLocks(db, { companyId: f.companyId, projectIds: [f.projectId!] }, async () => { + entered(); await pending; + }); + await started; + const restore = withExecutionWorkspaceLifecycleLock(db, f.workspaceId, () => + ensurePersistedExecutionWorkspaceAvailable({ db, base: f.base, workspace, + issue: { id: f.issueId, identifier: "GC-1", title: "Runtime restore" }, + agent: { id: null, name: "Fixture", companyId: f.companyId }, skipHostProvision: true, + prepareRestoredLocalSource: async sourceCwd => { + const identity = await fs.realpath(sourceCwd); + return async () => { expect(await fs.realpath(sourceCwd)).toBe(identity); }; + } })).then(value => ({ value }), error => ({ error })); + let restored: Awaited | undefined; + try { + await vi.waitFor(async () => { + const rows = await db.execute(sql`select count(*)::int n from pg_stat_activity where datname=current_database() and wait_event='advisory'`); + expect(rows[0].n).toBeGreaterThan(0); + }); + expect(await exists(f.cwd)).toBe(false); + } finally { release(); await sourceOwner; restored = await restore; } + expect(restored).not.toHaveProperty("error"); + expect(restored && "value" in restored ? restored.value?.cwd : null).toBe(f.cwd); + expect(git(f.nested.cwd, "rev-parse", "HEAD")).toBe(f.nestedA); + expect(await fs.readFile(f.nestedSettingsPath)).toEqual(f.settings); + expect((await fs.stat(f.nestedSettingsPath)).mode & 0o777).toBe(0o640); + }); + + it("takes the project fence before the lifecycle lock during an actual nested reopen", async () => { + const f = await nestedFixture(true, true); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: true, baseExpired: false }); + await fs.rm(f.sharedBase, { recursive: true }); + let entered!: () => void, release!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + const pending = new Promise(resolve => { release = resolve; }); + const owner = withWorkspaceSourceLocks(db, { companyId: f.companyId, projectIds: [f.projectId!] }, async () => { + entered(); await pending; + }); + await started; + const reopen = f.reopen().then(value => ({ value }), error => ({ error })); + let result: Awaited | undefined; + try { + await vi.waitFor(async () => { + const waiting = await db.execute(sql`select count(*)::int n from pg_stat_activity where datname=current_database() and wait_event='advisory'`); + expect(waiting[0].n).toBeGreaterThan(0); + }, { timeout: 5000 }); + const key = `execution_workspace_lifecycle:${f.workspaceId}`; + const granted = await db.execute(sql`select count(*)::int n from pg_locks where locktype='advisory' and granted + and classid=((hashtextextended(${key},0)>>32)&4294967295)::oid and objid=(hashtextextended(${key},0)&4294967295)::oid`); + expect(granted[0].n).toBe(0); + expect(await exists(f.cwd)).toBe(false); + } finally { release(); await owner; result = await reopen; } + expect(result).toMatchObject({ value: { ok: true, reopened: true } }); + expect(git(f.nested.cwd, "rev-parse", "HEAD")).toBe(f.nestedA); + expect(await fs.readFile(f.nestedSettingsPath)).toEqual(f.settings); + expect((await fs.stat(f.nestedSettingsPath)).mode & 0o777).toBe(0o640); + }); + it("expires a nested workspace and its unused private base with the generated ignore entry", async () => { const f = await nestedFixture(false); await f.advance(); diff --git a/server/src/routes/execution-workspaces.ts b/server/src/routes/execution-workspaces.ts index 836644182e1..ea37c280835 100644 --- a/server/src/routes/execution-workspaces.ts +++ b/server/src/routes/execution-workspaces.ts @@ -592,7 +592,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P }, run: async (reportProgress) => { const ensureWorkspaceAvailable = async () => - await ensurePersistedExecutionWorkspaceAvailable({ + await withExecutionWorkspaceLifecycleLock(db, existing.id, () => ensurePersistedExecutionWorkspaceAvailable({ base: { baseCwd: projectWorkspace?.cwd ?? workspaceCwd, source: existing.mode === "shared_workspace" ? "project_primary" : "task_session", @@ -633,7 +633,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P companyId: existing.companyId, }, recorder, - }); + })); if (action === "run") { if (!workspaceCommand || workspaceCommand.kind !== "job") { diff --git a/server/src/services/execution-workspace-lifecycle-lock.ts b/server/src/services/execution-workspace-lifecycle-lock.ts index 286dfc3f066..2f324d257ff 100644 --- a/server/src/services/execution-workspace-lifecycle-lock.ts +++ b/server/src/services/execution-workspace-lifecycle-lock.ts @@ -1,5 +1,6 @@ -import { withDedicatedDbConnection, type Db } from "@paperclipai/db"; -import { sql } from "drizzle-orm"; +import { executionWorkspaces, withDedicatedDbConnection, type Db } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { isolatedTaskSourceIdentity, managedProjectSourceIdentity, withWorkspaceSourceLocks } from "./workspace-source-lock.js"; import { withWorkspaceLockKeepalive } from "./native-runtime/native-workspace-lock-keepalive.js"; type Transaction = Parameters[0]>[0]; @@ -13,14 +14,32 @@ export async function acquireExecutionWorkspaceLifecycleLock( ); } -export function withExecutionWorkspaceLifecycleLock( +export async function withExecutionWorkspaceLifecycleLock( db: Db, workspaceId: string, work: (tx: Transaction) => Promise, ): Promise { - return withDedicatedDbConnection(db, (connection) => + const [source] = await db.select({ companyId: executionWorkspaces.companyId, projectId: executionWorkspaces.projectId, + providerType: executionWorkspaces.providerType, cwd: executionWorkspaces.cwd, providerRef: executionWorkspaces.providerRef }).from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId)); + const projectId = source && (source.cwd ?? source.providerRef) + ? await managedProjectSourceIdentity(source.companyId, (source.cwd ?? source.providerRef)!) : null; + const issueId = source && (source.cwd ?? source.providerRef) + ? await isolatedTaskSourceIdentity(source.companyId, (source.cwd ?? source.providerRef)!) : null; + const projectIds = [...new Set([projectId, source && ["local_fs", "git_worktree"].includes(source.providerType) + ? source.projectId : null].filter((id): id is string => Boolean(id)))]; + const workWithLifecycle = () => withDedicatedDbConnection(db, (connection) => connection.transaction(async (tx) => { await acquireExecutionWorkspaceLifecycleLock(tx, workspaceId); + if ((projectIds.length || issueId) && source) { + const [fresh] = await tx.select({ companyId: executionWorkspaces.companyId, projectId: executionWorkspaces.projectId, + providerType: executionWorkspaces.providerType, cwd: executionWorkspaces.cwd, providerRef: executionWorkspaces.providerRef }).from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId)); + if (!fresh || fresh.companyId !== source.companyId || fresh.projectId !== source.projectId + || fresh.providerType !== source.providerType || !(fresh.cwd ?? fresh.providerRef) + || await managedProjectSourceIdentity(fresh.companyId, (fresh.cwd ?? fresh.providerRef)!) !== projectId + || await isolatedTaskSourceIdentity(fresh.companyId, (fresh.cwd ?? fresh.providerRef)!) !== issueId) { + throw new Error("workspace_source_scope_changed"); + } + } const rows = await tx.execute( sql`select setting::bigint as idle_timeout_ms from pg_settings where name = 'idle_in_transaction_session_timeout'`, ); @@ -31,4 +50,12 @@ export function withExecutionWorkspaceLifecycleLock( ); }), ); + return (projectIds.length || issueId) && source + ? withWorkspaceSourceLocks(db, { companyId: source.companyId, projectIds, issueIds: issueId ? [issueId] : [] }, workWithLifecycle) + : workWithLifecycle(); +} + +export async function tryAcquireExecutionWorkspaceLifecycleLock(tx: Transaction, workspaceId: string) { + const rows = await tx.execute(sql`select pg_try_advisory_xact_lock(hashtextextended(${`execution_workspace_lifecycle:${workspaceId}`}, 0)) as acquired`); + return rows[0]?.acquired === true; } diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 3b29a35aa28..69f9c6dd163 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -8,6 +8,7 @@ import { measureRunTransaction } from "./execution-transaction-timing.js"; import { measureRunPhase } from "./execution-preparation-timing.js"; import { assertExecutionWorkspaceRecordAccess, assertTaskWorkspaceExecutionAccess, assertTaskWorkspaceSourceProjectAccess } from "./task-workspace-access.js"; import { verifiedRecordedIsolatedRemoteSource, withIsolatedRepositorySourceLock } from "./isolated-remote-workspace-source.js"; +import { managedProjectSourceIdentity, retainCurrentWorkspaceSource, withWorkspaceSourceLocks } from "./workspace-source-lock.js"; import { expireReconstructableGitWorkspace, gitWorkspaceLastActivityAt, hasPendingGitExpiration, listIssuesUsingGitWorkspace } from "./git-workspace-expiration.js"; import { expireIsolatedTaskDirectory, hasPendingTaskDirectoryRetention, isIsolatedTaskDirectoryWorkspace, isRetryableTaskDirectorySkip, mayBeIsolatedTaskDirectoryWorkspace, readTaskDirectoryRetention, restoreIsolatedTaskDirectory, supersedeTaskDirectoryRestorePoint, @@ -4318,6 +4319,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic return { ok: true, reopened: true, workspace: toExecutionWorkspace(active!), generation: generation + 1 }; }); } + const reopenSource = async (): Promise => { const snapshot = freshSnapshot.providerType === "plugin" ? freshSnapshot : await rehomeExpiredIsolatedRestoration(db, freshSnapshot); if (snapshot.providerType === "plugin") { return withExecutionWorkspaceLifecycleLock(db, snapshot.id, async tx => { @@ -4375,6 +4377,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic ]); let resolvedBaseCwd: string | null; try { + if (projectWorkspace?.cwd) await retainCurrentWorkspaceSource(projectWorkspace.cwd); + if (recipe?.sourcePath) await retainCurrentWorkspaceSource(recipe.sourcePath); resolvedBaseCwd = recipe && isClosedExecutionWorkspaceStatus(snapshot.status) ? await restoreGitCacheBase(snapshot, recipe, gitAuth, undefined, { freshStart }) : await verifiedRecordedIsolatedRemoteSource(snapshot) ?? projectWorkspace?.cwd ?? null; @@ -4769,6 +4773,16 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic }); return result; }; + if (freshSnapshot.providerType === "plugin") return reopenSource(); + const projectIds = new Set(); + if (freshSnapshot.projectId) projectIds.add(freshSnapshot.projectId); + for (const cwd of new Set([freshSnapshot.cwd, freshSnapshot.providerRef])) { + if (!cwd) continue; + const projectId = await managedProjectSourceIdentity(freshSnapshot.companyId, cwd); + if (projectId) projectIds.add(projectId); + } + return withWorkspaceSourceLocks(db, { companyId: freshSnapshot.companyId, projectIds: [...projectIds] }, reopenSource); + }; return authorizedSnapshot.sourceIssueId ? withIsolatedRepositorySourceLock(db, authorizedSnapshot.companyId, authorizedSnapshot.sourceIssueId, reopen) : reopen(); diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index 136775ebbf4..3291dd57072 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -44,6 +44,8 @@ import { assertNativeSessionPreparationAuthority } from "../native-runtime/nativ import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js"; import { projectService } from "../projects.js"; import { WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, WorkspaceBusyDeferral } from "./retries.js"; +import { withWorkspaceSourceLocks } from "../workspace-source-lock.js"; +import { resolveManagedProjectWorkspaceDir } from "../../home-paths.js"; const execFile = promisify(execFileCallback); const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_WORKSPACE_PREPARATION_TEST_DATABASE_URL"); @@ -126,6 +128,31 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace }; return { input, source, issue, run, envOrchestrator }; } + it("waits for the actual managed project source fence before admitting a new issue", async () => { + const f = await fixture(); + const companyId = f.input.agent.companyId, projectId = f.issue.projectId!; + const cwd = resolveManagedProjectWorkspaceDir({ companyId, projectId }); + await mkdir(cwd, { recursive: true }); + await writeFile(join(cwd, "legacy.txt"), "legacy bytes remain unchanged\n"); + await db.update(projectWorkspaces).set({ cwd, repoUrl: null }) + .where(eq(projectWorkspaces.id, f.issue.projectWorkspaceId!)); + let entered!: () => void, release!: () => void, prepared = false; + const started = new Promise(resolve => { entered = resolve; }); + const pending = new Promise(resolve => { release = resolve; }); + const owner = withWorkspaceSourceLocks(db, { companyId, projectIds: [projectId] }, async () => { entered(); await pending; }); + await started; + const preparation = prepareHeartbeatWorkspace(db, f.input).then(value => { prepared = true; return value; }); + try { + await vi.waitFor(async () => { + const rows = await db.execute(sql`select count(*)::int n from pg_stat_activity where datname=current_database() and wait_event='advisory'`); + expect(rows[0].n).toBeGreaterThan(0); + }); + expect(prepared).toBe(false); + expect(await readFile(join(cwd, "legacy.txt"), "utf8")).toBe("legacy bytes remain unchanged\n"); + } finally { release(); await owner; } + await expect(preparation).resolves.toMatchObject({ executionWorkspace: expect.objectContaining({ cwd }) }); + expect(prepared).toBe(true); + }); it.each([false, true])("places Cursor Cloud in its provider API without a Computer under managed-only policy (ephemeral: %s)", async ephemeral => { vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", String(ephemeral)); const { input, source, issue, run } = await fixture(); @@ -554,8 +581,8 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace input.environment.selectedEnvironmentId = environment.id; input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = false; input.task.issueRef = { ...issue, executionWorkspaceId: imported.id }; - input.reuse.existingExecutionWorkspace = toExecutionWorkspace(imported); - input.reuse.reusableExistingExecutionWorkspace = toExecutionWorkspace(imported); + input.reuse.existingExecutionWorkspace = toExecutionWorkspace(sourceKind === "path" ? original : imported); + input.reuse.reusableExistingExecutionWorkspace = toExecutionWorkspace(sourceKind === "path" ? original : imported); input.reuse.requestedShouldReuseExisting = true; input.reuse.workspaceReuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: imported.id, issueExecutionWorkspacePreference: null, existingExecutionWorkspaceStatus: imported.status }); diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index 125cce63165..de9683f30cd 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -47,7 +47,7 @@ import { readNativeWorkspaceSyncReference, readNativeWorkspaceSyncLocalCwd } fro import { parseObject } from "../../adapters/utils.js"; import { materializeNativeChatTaskRoot } from "../native-runtime/native-chat-workspace.js"; import { materializeIsolatedTaskDirectory } from "../isolated-task-directory.js"; -import { withIsolatedRepositorySourceLock } from "../isolated-remote-workspace-source.js"; +import { withWorkspaceSourceLocks } from "../workspace-source-lock.js"; import { cleanupExecutionWorkspaceArtifacts, ensurePersistedExecutionWorkspaceAvailable, @@ -170,9 +170,23 @@ function readNonEmptyString(value: unknown): string | null { export async function prepareHeartbeatWorkspace(db: Db, input: HeartbeatWorkspacePreparationInput) { const sourceIssueId = input.reuse.reusableExistingExecutionWorkspace?.sourceIssueId ?? input.reuse.existingExecutionWorkspace?.sourceIssueId ?? input.task.issueId; - if (!sourceIssueId || input.reuse.isDotRun || resolveProviderApiPlacement(input)) return prepareHeartbeatWorkspaceLocked(db, input); - return withIsolatedRepositorySourceLock(db, input.agent.companyId, sourceIssueId, + if (input.reuse.isDotRun || resolveProviderApiPlacement(input)) return prepareHeartbeatWorkspaceLocked(db, input); + if (!sourceIssueId) return withWorkspaceSourceLocks(db, { companyId: input.agent.companyId }, () => prepareHeartbeatWorkspaceLocked(db, input)); + return withWorkspaceSourceLocks(db, { companyId: input.agent.companyId, + issueIds: [...new Set([sourceIssueId, input.task.issueId].filter((id): id is string => Boolean(id)))] }, async () => { + const reuse = { ...input.reuse }; + for (const key of ["existingExecutionWorkspace", "reusableExistingExecutionWorkspace"] as const) { + const selected = reuse[key]; + if (!selected || readRemoteWorkspaceAuthority(selected.metadata)) continue; + const [fresh] = await db.select({ metadata: executionWorkspacesTable.metadata }).from(executionWorkspacesTable) + .where(and(eq(executionWorkspacesTable.companyId, input.agent.companyId), eq(executionWorkspacesTable.id, selected.id))); + if (!readRemoteWorkspaceAuthority(fresh?.metadata)) continue; + const current = await input.services.executionWorkspacesSvc.getStoredById(input.agent.companyId, selected.id); + if (current && readRemoteWorkspaceAuthority(current.metadata)) reuse[key] = current; + } + return prepareHeartbeatWorkspaceLocked(db, { ...input, reuse }); + }); } async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspacePreparationInput) { diff --git a/server/src/services/heartbeat/workspaces.ts b/server/src/services/heartbeat/workspaces.ts index d9787c96580..49a1e3e1f3f 100644 --- a/server/src/services/heartbeat/workspaces.ts +++ b/server/src/services/heartbeat/workspaces.ts @@ -1,3 +1,4 @@ +import { retainCurrentWorkspaceSource } from "../workspace-source-lock.js"; import { effectiveProjectWorkspacePolicy } from "../project-workspace-provider.js"; import { canonicalRepositoryRoot } from "../execution-workspace-source-roots.js"; import { assertTaskWorkspaceExecutionAccess } from "../task-workspace-access.js"; @@ -589,6 +590,7 @@ async function resolveConfiguredOrManagedProjectCwd(input: { }): Promise<{ cwd: string; warning: string | null }> { const configuredCwd = readNonEmptyString(input.cwd); if (configuredCwd && configuredCwd !== REPO_ONLY_CWD_SENTINEL) { + await retainCurrentWorkspaceSource(configuredCwd); return { cwd: configuredCwd, warning: null }; } return ensureManagedProjectWorkspace({ @@ -3399,6 +3401,7 @@ export function createHeartbeatWorkspaceResolver(db: Db) { managedBase && workspace.repoUrl ? `${managedBase}-${createHash("sha256").update(workspace.repoUrl).digest("hex").slice(0, 12)}` : null, ].filter((value): value is string => Boolean(value)); + await retainCurrentWorkspaceSource(workspace.cwd ?? workspace.providerRef ?? "./"); const cwd = await resolvePersistedGitWorkspaceSource({ companyId: agent.companyId, projectId: sourceProjectId, @@ -3653,6 +3656,7 @@ export function createHeartbeatWorkspaceResolver(db: Db) { .then((stats) => stats.isDirectory()) .catch(() => false); if (sessionCwdExists) { + await retainCurrentWorkspaceSource(sessionCwd!); return { cwd: sessionCwd, source: "task_session" as const, @@ -3674,6 +3678,7 @@ export function createHeartbeatWorkspaceResolver(db: Db) { const cwd = configuredCwd ? path.resolve(configuredCwd) : issueId ? await materializeIsolatedTaskDirectory({ companyId: agent.companyId, issueId }) : resolveDefaultAgentWorkspaceDir(agent.id); + await retainCurrentWorkspaceSource(cwd); await fs.mkdir(cwd, { recursive: true }); const warnings: string[] = []; if (sessionCwd && sessionCwdLooksUnsafe) { @@ -3730,6 +3735,7 @@ export function createHeartbeatWorkspaceResolver(db: Db) { previousSessionParams, opts, ); + await retainCurrentWorkspaceSource(anchor.cwd); if (!isMultiProjectWorkspaceSyncEnabled()) { return { ...anchor, diff --git a/server/src/services/isolated-remote-workspace-source.ts b/server/src/services/isolated-remote-workspace-source.ts index 44e7c6ae7db..b3014b7d3ee 100644 --- a/server/src/services/isolated-remote-workspace-source.ts +++ b/server/src/services/isolated-remote-workspace-source.ts @@ -1,6 +1,4 @@ -import { AsyncLocalStorage } from "node:async_hooks"; -import { withDedicatedDbConnection, type Db } from "@paperclipai/db"; -import { sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; import path from "node:path"; import fs from "node:fs/promises"; import { execFile as execFileCallback } from "node:child_process"; @@ -11,32 +9,10 @@ import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { materializeManagedProjectWorkspace } from "./managed-repository-checkout.js"; import { measureRunPhase } from "./execution-preparation-timing.js"; import { isRestrictedWorkspaceSourceDenial } from "./workspace-source-restriction.js"; -import { withWorkspaceLockKeepalive } from "./native-runtime/native-workspace-lock-keepalive.js"; const execFile = promisify(execFileCallback); -const sourceLocks = new AsyncLocalStorage>(); -export class IsolatedRepositorySourceBusy extends Error {} - -/** Creation, reopening and base expiration share this cross-process exclusion. */ -export async function withIsolatedRepositorySourceLock(db: Db, companyId: string, issueId: string, - action: () => Promise, tryOnly = false): Promise { - const key = `isolated_repository_source:${companyId}:${issueId}`; - const held = sourceLocks.getStore() ?? []; - if (held.some(lock => lock.db === db && lock.key === key)) return action(); - return withDedicatedDbConnection(db, connection => connection.transaction(async tx => { - const acquire = tryOnly - ? sql`pg_try_advisory_xact_lock(hashtextextended(${key}, 0))` - : sql`pg_advisory_xact_lock(hashtextextended(${key}, 0))`; - const rows = await tx.execute(sql`select ${acquire} as acquired, - (select setting::bigint from pg_settings where name = 'idle_in_transaction_session_timeout') as idle_timeout_ms`); - if (tryOnly && !rows[0]?.acquired) throw new IsolatedRepositorySourceBusy(); - return withWorkspaceLockKeepalive(tx, Number(rows[0]?.idle_timeout_ms), - () => sourceLocks.run([...held, { db, key }], action)); - })); -} - - +export { IsolatedRepositorySourceBusy, withIsolatedRepositorySourceLock } from "./workspace-source-lock.js"; export function isolatedRemoteSourcePath(companyId: string, issueId: string): string { if ([companyId, issueId].some(id => !/^[a-zA-Z0-9_-]+$/.test(id))) { throw new Error("Invalid isolated repository source identity"); diff --git a/server/src/services/isolated-repository-rehome.ts b/server/src/services/isolated-repository-rehome.ts index 6314f5edcc7..c11ba8e0b60 100644 --- a/server/src/services/isolated-repository-rehome.ts +++ b/server/src/services/isolated-repository-rehome.ts @@ -9,6 +9,7 @@ import { executionWorkspaces, projectWorkspaces, workspaceRuntimeServices, heart withDedicatedDbConnection, type Db } from "@paperclipai/db"; import { PROJECT_REPOSITORIES_DIR } from "@paperclipai/adapter-utils/git-workspace-sync"; import { IsolatedRepositorySourceBusy, isolatedRemoteSourcePath, verifiedRecordedIsolatedRemoteSource, withIsolatedRepositorySourceLock } from "./isolated-remote-workspace-source.js"; +import { retainCurrentWorkspaceSource, withWorkspaceSourceLocks } from "./workspace-source-lock.js"; import { acquireGitExpirationLocks, recoverGitExpirationLocks, type GitExpirationLockLocation } from "./git-workspace-expiration-locks.js"; import { allocatedBytes, describeGitExpirationFailure, expirationScope, gitExpirationSkipReason, withAnonymousGitFallback, type GitExpirationFailure } from "./git-workspace-expiration.js"; import { finishGitCacheTrash, hasCommitsMissingUpstream, managedSourcePaths, readGitCacheExpiration, readGitCacheRestoration, verifiedGitCacheSource, @@ -353,13 +354,17 @@ async function undoMove(db: Db | Tx, workspaceId: string, pending: IsolatedRepos export function withWorkspaceMoveLocks(db: Db, workspace: Workspace, work: (tx: Tx) => Promise) { return withIsolatedRepositorySourceLock(db, workspace.companyId, workspace.sourceIssueId!, () => - withDedicatedDbConnection(db, connection => connection.transaction(async tx => { - const rows = await tx.execute<{ locked: boolean; idle_timeout_ms: string | number | null }>(sql`select - pg_try_advisory_xact_lock(hashtextextended(${`execution_workspace_lifecycle:${workspace.id}`}, 0)) as locked, - (select setting::bigint from pg_settings where name = 'idle_in_transaction_session_timeout') as idle_timeout_ms`); - if (!rows[0]?.locked) throw new Error("lifecycle_busy"); - return withWorkspaceLockKeepalive(tx, Number(rows[0]?.idle_timeout_ms), () => work(tx)); - })), true); + withWorkspaceSourceLocks(db, { companyId: workspace.companyId, projectIds: workspace.projectId ? [workspace.projectId] : [] }, async () => { + const cwd = workspace.cwd ?? workspace.providerRef; + if (cwd) await retainCurrentWorkspaceSource(cwd); + return withDedicatedDbConnection(db, connection => connection.transaction(async tx => { + const rows = await tx.execute<{ locked: boolean; idle_timeout_ms: string | number | null }>(sql`select + pg_try_advisory_xact_lock(hashtextextended(${`execution_workspace_lifecycle:${workspace.id}`}, 0)) as locked, + (select setting::bigint from pg_settings where name = 'idle_in_transaction_session_timeout') as idle_timeout_ms`); + if (!rows[0]?.locked) throw new Error("lifecycle_busy"); + return withWorkspaceLockKeepalive(tx, Number(rows[0]?.idle_timeout_ms), () => work(tx)); + })); + }), true); } function emptyResult(): IsolatedRepositoryRehomeResult { diff --git a/server/src/services/legacy-workspace-recovery-migration.ts b/server/src/services/legacy-workspace-recovery-migration.ts index 5fd3426503c..51c05ee1e60 100644 --- a/server/src/services/legacy-workspace-recovery-migration.ts +++ b/server/src/services/legacy-workspace-recovery-migration.ts @@ -8,10 +8,11 @@ export type LegacyWorkspaceRecoverySource = { runId: string; issueId: string; reference: NativeWorkspaceSyncReference | null; recoveryAllocations?: { leaseId: string; providerLeaseId: string; remoteCwd: string }[] | null; unresolvedFinalization?: { phase: string; leaseOwner: string | null; leaseExpiresAt: string | null }; + onlineProof?: { schema: "paperclip.online-legacy-workspace-preservation.v1"; fingerprint: string }; }; export const LEGACY_WORKSPACE_RECOVERY_MIGRATION = "paperclip.legacy-workspace-recovery-migration.v1"; -/** Offline migration preserves an unresolved coordinator, but it cannot replay +/** Migration preserves an unresolved coordinator, but it cannot replay * an old execution against a local copyback destination that is no longer its * authority. File recovery and explicit task continuation remain available. */ export function legacyWorkspaceRecoveryNotMigrated() { diff --git a/server/src/services/managed-repository-checkout.ts b/server/src/services/managed-repository-checkout.ts index b077b9ab55f..cbd8ad0b1d2 100644 --- a/server/src/services/managed-repository-checkout.ts +++ b/server/src/services/managed-repository-checkout.ts @@ -1,3 +1,4 @@ +import { retainCurrentWorkspaceSource } from "./workspace-source-lock.js"; import fs from "node:fs/promises"; import path from "node:path"; import { execFile as execFileCallback } from "node:child_process"; @@ -67,6 +68,9 @@ export async function materializeManagedProjectWorkspace( objectCacheCompanyDirectory?: string; }, ): Promise<{ cwd: string; warning: string | null }> { + await retainCurrentWorkspaceSource(cwd); + if (input.localSource) await retainCurrentWorkspaceSource(input.localSource); + if (input.gitObjectSource) await retainCurrentWorkspaceSource(input.gitObjectSource.cwd); if (input.localSource && input.gitObjectSource) throw new Error("Conflicting managed checkout sources"); await fs.mkdir(path.dirname(cwd), { recursive: true }); const stats = await fs.stat(cwd).catch(() => null); diff --git a/server/src/services/offline-workspace-operation-reconciliation.test.ts b/server/src/services/offline-workspace-operation-reconciliation.test.ts index d7667d177cf..fa9a41acad4 100644 --- a/server/src/services/offline-workspace-operation-reconciliation.test.ts +++ b/server/src/services/offline-workspace-operation-reconciliation.test.ts @@ -5,9 +5,10 @@ import path from "node:path"; import { eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { activityLog, agents, closeRegisteredClients, companies, completionContracts, createDb, executionWorkspaces, - heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, workspaceOperations } from "@paperclipai/db"; + heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults, workspaceOperations } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; -import { reconcileOfflineWorkspaceOperations } from "./offline-workspace-operation-reconciliation.js"; +import { reconcileOfflineWorkspaceOperations, reconcileOnlineWorkspaceOperations } from "./offline-workspace-operation-reconciliation.js"; +import { adapterExecutionControls, createAdapterExecutionControl } from "./adapter-execution-control.js"; import { inventoryRemoteWorkspaceMigration, migrateRemoteWorkspace } from "./remote-workspace-migration.js"; import { createStorageService } from "../storage/service.js"; import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; @@ -228,4 +229,78 @@ describe.skipIf(!support.supported)("offline workspace operation receipts", () = await db.execute(sql`drop function reject_operation_audit()`); } }); + + it("reconciles exact superseded receipts online while unrelated work, held files, and task intent remain", async () => { + const f = await fixture(), other = await fixture(), root = await mkdtemp(path.join(os.tmpdir(), "online-receipt-files-")); + const [oldRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [active] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: oldRun.agentId, + status: "running", runtimeMode: "native", controllerLeaseExpiresAt: new Date(Date.now() + 60_000), + runnerProfileJson: { remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1" } } }).returning(); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, + kind: "configuration_validation", cause: "configuration_incomplete", fingerprint: randomUUID(), nextAction: "Keep the warning" }); + const retained = async () => ({ + runs: await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)), + owners: await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.companyId, f.companyId)), + issues: await db.select().from(issues).where(eq(issues.companyId, f.companyId)), + actions: await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId)), + }); + try { + await writeFile(path.join(root, "saved.txt"), "Unpublished user files\n"); + await db.update(executionWorkspaces).set({ cwd: root }).where(eq(executionWorkspaces.id, f.workspaceId)); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "false"); + const before = await retained(), receipt = await operation(f.receiptId), otherAttempt = await operation(other.operationId); + expect(await reconcileOnlineWorkspaceOperations(db, { companyId: f.companyId, signal: new AbortController().signal })) + .toMatchObject({ inspected: 1, reconciled: 1, held: [] }); + expect(await operation(f.operationId)).toMatchObject({ status: "failed", metadata: { original: "keep", + onlineReconciliation: { schema: "paperclip.online-workspace-operation-reconciliation.v1", + reason: "superseded_workspace_finalization", supersedingOperationId: f.receiptId, resultId: f.resultId } } }); + expect(await operation(f.receiptId)).toEqual(receipt); + expect(await operation(other.operationId)).toEqual(otherAttempt); + expect(await retained()).toEqual(before); + expect(await readFile(path.join(root, "saved.txt"), "utf8")).toBe("Unpublished user files\n"); + expect(await db.select().from(activityLog).where(eq(activityLog.entityId, f.operationId))).toHaveLength(1); + } finally { + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, active.id)); + await rm(root, { recursive: true, force: true }); + } + }); + + it.each(["expired_owner", "physical_owner", "adapter_control", "uncertain", "worktree_prepare"])("keeps an online %s obligation held", async kind => { + const f = await fixture(); + if (kind === "expired_owner") await db.update(nativeRunFinalizations).set({ leaseOwner: "surviving-owner", leaseExpiresAt: new Date(0) }).where(eq(nativeRunFinalizations.runId, f.runId)); + if (kind === "physical_owner") await db.update(heartbeatRuns).set({ runnerProfileJson: { + ...f.profile, nativeWorkspaceFinalizationOwner: { token: "unverified-physical-owner" } } }).where(eq(heartbeatRuns.id, f.runId)); + if (kind === "adapter_control") adapterExecutionControls.set(f.runId, createAdapterExecutionControl()); + if (kind === "uncertain") await db.update(workspaceOperations).set({ metadata: { ...f.metadata, uncertain: true } }).where(eq(workspaceOperations.id, f.operationId)); + if (kind === "worktree_prepare") await db.update(workspaceOperations).set({ phase: "worktree_prepare" }).where(eq(workspaceOperations.id, f.operationId)); + const before = await operation(f.operationId), receipt = await operation(f.receiptId); + try { + expect(await reconcileOnlineWorkspaceOperations(db, { companyId: f.companyId, signal: new AbortController().signal })) + .toMatchObject({ inspected: 1, reconciled: 0, held: [{ operationId: f.operationId }] }); + expect(await operation(f.operationId)).toEqual(before); + expect(await operation(f.receiptId)).toEqual(receipt); + expect(await db.select().from(activityLog).where(eq(activityLog.entityId, f.operationId))).toHaveLength(0); + } finally { adapterExecutionControls.delete(f.runId); } + }); + + it("keeps the online attempt unchanged while a workspace publication owns its row", async () => { + const f = await fixture(); + let release!: () => void, ready!: () => void; + const barrier = new Promise(resolve => { release = resolve; }), locked = new Promise(resolve => { ready = resolve; }); + const publisher = db.transaction(async tx => { + await tx.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1" } } }) + .where(eq(executionWorkspaces.id, f.workspaceId)); + ready(); await barrier; + }); + await locked; + const before = await operation(f.operationId); + try { + expect(await reconcileOnlineWorkspaceOperations(db, { companyId: f.companyId, signal: new AbortController().signal })) + .toMatchObject({ reconciled: 0, held: [{ operationId: f.operationId, reason: "execution_rows_busy" }] }); + expect(await operation(f.operationId)).toEqual(before); + } finally { release(); await publisher; } + expect(await reconcileOnlineWorkspaceOperations(db, { companyId: f.companyId, signal: new AbortController().signal })) + .toMatchObject({ inspected: 0, reconciled: 0 }); + }); }); diff --git a/server/src/services/offline-workspace-operation-reconciliation.ts b/server/src/services/offline-workspace-operation-reconciliation.ts index d8121342955..6b1f67329f8 100644 --- a/server/src/services/offline-workspace-operation-reconciliation.ts +++ b/server/src/services/offline-workspace-operation-reconciliation.ts @@ -1,12 +1,14 @@ import { and, asc, desc, eq, gt, inArray, isNotNull, sql } from "drizzle-orm"; import { executionWorkspaces, heartbeatRuns, nativeRunFinalizations, nativeRunResults, workspaceOperations, type Db } from "@paperclipai/db"; -import { persistActivity } from "./activity-log.js"; +import { persistActivity, publishActivity } from "./activity-log.js"; +import { adapterExecutionControls } from "./adapter-execution-control.js"; import { offlineWorkspaceMigrationBlockers } from "./remote-workspace-migration-tree.js"; import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js"; import { tryLockNativeWorkspaceFinalization } from "./native-runtime/native-workspace-finalization-ownership.js"; type Candidate = typeof workspaceOperations.$inferSelect; type Held = { operationId: string; reason: string }; +type ReconciliationInput = { mode: "offline" | "online"; writersStopped: boolean; signal: AbortSignal; companyId?: string }; const unresolved = () => inArray(workspaceOperations.status, ["running", "pending"]); const localMigrationSource = () => and(isNotNull(executionWorkspaces.cwd), inArray(executionWorkspaces.providerType, ["local_fs", "git_worktree"]), @@ -25,16 +27,17 @@ function databaseCode(error: unknown): string | undefined { return value?.code ?? (value?.cause ? databaseCode(value.cause) : undefined); } -async function reconcileAttempt(db: Db, candidate: Candidate, input: { writersStopped: boolean; signal: AbortSignal }) { +async function reconcileAttempt(db: Db, candidate: Candidate, input: ReconciliationInput) { const held = (reason: string): Held => ({ operationId: candidate.id, reason }); const { heartbeatRunId: runId, issueId, executionWorkspaceId: workspaceId } = candidate; if (candidate.phase !== "workspace_finalize" || !runId || !issueId || !workspaceId) return held("superseding_finalization_receipt_missing"); + let publication: Awaited>["publication"] | undefined; try { - return await db.transaction(async tx => { + const outcome = await db.transaction(async tx => { const database = tx as unknown as Db; input.signal.throwIfAborted(); - await assertOffline(database, input.writersStopped); + if (input.mode === "offline") await assertOffline(database, input.writersStopped); if (!await tryLockNativeWorkspaceFinalization(database, { companyId: candidate.companyId, runId })) return held("finalization_owner_busy"); // Follow the finalizer's coordinator-before-run order. NOWAIT keeps an @@ -48,6 +51,7 @@ async function reconcileAttempt(db: Db, candidate: Candidate, input: { writersSt )).for("update", { noWait: true }); if (!run || run.runtimeMode !== "native" || run.status !== "succeeded" || (run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date()) + || adapterExecutionControls.has(run.id) || run.runnerProfileJson?.remoteWorkspace != null || run.runnerProfileJson?.nativeWorkspaceFinalizationOwner != null || !coordinator || coordinator.phase !== "committed" || !coordinator.resultId || coordinator.leaseOwner !== null || coordinator.leaseExpiresAt !== null || coordinator.nextAttemptAt !== null) return held("committed_finalization_proof_missing"); @@ -73,7 +77,8 @@ async function reconcileAttempt(db: Db, candidate: Candidate, input: { writersSt // These fields represent independent obligations. A successful later // export does not authorize dropping them or rewriting earlier history. if (current.metadata?.uncertain === true || current.metadata?.uncertain === "true" - || current.metadata?.saveWork != null || current.metadata?.offlineReconciliation != null) + || current.metadata?.saveWork != null || current.metadata?.offlineReconciliation != null + || current.metadata?.onlineReconciliation != null) return held("independent_operation_obligation"); const [receipt] = await tx.select().from(workspaceOperations).where(and( eq(workspaceOperations.companyId, candidate.companyId), eq(workspaceOperations.executionWorkspaceId, workspaceId), @@ -86,36 +91,34 @@ async function reconcileAttempt(db: Db, candidate: Candidate, input: { writersSt if (!receipt || receipt.metadata?.uncertain === true || receipt.metadata?.uncertain === "true") return held("superseding_finalization_receipt_missing"); input.signal.throwIfAborted(); - await assertOffline(database, input.writersStopped); + if (input.mode === "offline") await assertOffline(database, input.writersStopped); const now = new Date(); - const provenance = { schema: "paperclip.offline-workspace-operation-reconciliation.v1", + const provenance = { schema: input.mode === "offline" ? "paperclip.offline-workspace-operation-reconciliation.v1" + : "paperclip.online-workspace-operation-reconciliation.v1", reason: "superseded_workspace_finalization", reconciledAt: now.toISOString(), previousStatus: current.status, supersedingOperationId: receipt.id, runId, resultId: result.id, descriptorSha256: reference.descriptorSha256, finalHostSha256: reference.finalHostSha256 }; await tx.update(workspaceOperations).set({ status: "failed", finishedAt: now, updatedAt: now, - metadata: { ...current.metadata, offlineReconciliation: provenance } }).where(and( + metadata: { ...current.metadata, [input.mode === "offline" ? "offlineReconciliation" : "onlineReconciliation"]: provenance } }).where(and( eq(workspaceOperations.id, current.id), eq(workspaceOperations.companyId, current.companyId), unresolved(), )); - // No live publication is needed in the stopped maintenance process. The - // audit and the failed attempt commit (or roll back) together. - await persistActivity(database, { companyId: current.companyId, actorType: "system", actorId: "ephemeral-migration", + const activity = await persistActivity(database, { companyId: current.companyId, actorType: "system", + actorId: input.mode === "offline" ? "ephemeral-migration" : "legacy-workspace-preservation", action: "workspace_operation.reconciled", entityType: "workspace_operation", entityId: current.id, runId, issueId, details: provenance }); + publication = activity.publication; return null; }); + if (input.mode === "online" && publication) publishActivity(publication); + return outcome; } catch (error) { if (databaseCode(error) === "55P03") return held("execution_rows_busy"); throw error; } } -/** Reconcile receipts only after Cloud has stopped all old application and - * worker deployments. This never executes workspace work, declares success, - * clears execution authority, or retires a recovery/source obligation. */ -export async function reconcileOfflineWorkspaceOperations(db: Db, input: { - writersStopped: boolean; signal: AbortSignal; companyId?: string; -}) { - await assertOffline(db, input.writersStopped); +async function reconcileWorkspaceOperations(db: Db, input: ReconciliationInput) { + if (input.mode === "offline") await assertOffline(db, input.writersStopped); let after: string | undefined, inspected = 0, reconciled = 0; const held: Held[] = []; for (;;) { @@ -136,3 +139,19 @@ export async function reconcileOfflineWorkspaceOperations(db: Db, input: { after = rows.at(-1)!.operation.id; } } + +/** Reconcile receipts only after Cloud has stopped all old application and + * worker deployments. This never executes workspace work, declares success, + * clears execution authority, or retires a recovery/source obligation. */ +export async function reconcileOfflineWorkspaceOperations(db: Db, input: { + writersStopped: boolean; signal: AbortSignal; companyId?: string; +}) { + return reconcileWorkspaceOperations(db, { ...input, mode: "offline" }); +} + +export async function reconcileOnlineWorkspaceOperations(db: Db, input: { + companyId: string; signal: AbortSignal; +}) { + if (!input.companyId) throw new Error("online_reconciliation_company_required"); + return reconcileWorkspaceOperations(db, { ...input, mode: "online", writersStopped: false }); +} diff --git a/server/src/services/online-legacy-workspace-recovery.test.ts b/server/src/services/online-legacy-workspace-recovery.test.ts new file mode 100644 index 00000000000..fffe358b444 --- /dev/null +++ b/server/src/services/online-legacy-workspace-recovery.test.ts @@ -0,0 +1,213 @@ +import { randomUUID } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, closeRegisteredClients, companies, createDb, environmentLeases, environments, + executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { inspectOnlineLegacyWorkspaceRecoveryHold, inspectOnlineLegacyWorkspaceRecoverySource, + lockOnlineLegacyWorkspaceRecoverySources } from "./online-legacy-workspace-recovery.js"; +import { tryLockNativeWorkspaceFinalization } from "./native-runtime/native-workspace-finalization-ownership.js"; +import { nativeSha256 } from "./native-runtime/canonical.js"; + +const support = await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("online legacy source preservation", () => { + let database: Awaited>, db: ReturnType; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("online-legacy-preservation-"); + db = createDb(database.connectionString); + }, 90_000); + afterAll(async () => { + if (database) await closeRegisteredClients(database.connectionString); + await database?.cleanup(); + }); + async function fixture() { + const companyId = randomUUID(), workspaceId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Legacy preservation", issuePrefix: `P${companyId.slice(0, 6)}` }); + const [agent] = await db.insert(agents).values({ companyId, name: "Historical native worker", adapterType: "paperclip_runner" }).returning(); + const [issue] = await db.insert(issues).values({ companyId, title: "Preserve old files", status: "blocked" }).returning(); + const [workspace] = await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, + sourceIssueId: issue.id, providerType: "local_fs", mode: "isolated_workspace", strategyType: "project_primary", name: "Legacy files" }).returning(); + const reference = { schema: "paperclip.native-workspace-sync/v3" as const, state: "prepared" as const, + workspaceId, leaseId: randomUUID(), providerLeaseId: randomUUID(), remoteCwd: "/workspaces/retained", + descriptorSha256: "12".repeat(32), baselineSha256: "34".repeat(32), resourceDisposition: "destroy" as const }; + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId: agent.id, nativeIssueId: issue.id, + status: "failed", runtimeMode: "native", processPid: 4321, processGroupId: 4321, + processStartedAt: new Date("2026-01-01T00:00:00Z"), runnerProfileJson: { nativeWorkspaceSync: reference }, + resultJson: { checkpoint: "preserve this result", workspaceRestoreRecovery: { + schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [reference.leaseId] } } }); + await db.insert(nativeRunFinalizations).values({ companyId, issueId: issue.id, runId, phase: "terminal_failure", + failureCode: "native_workspace_sync_out_retry_exhausted", recoveryHistory: [{ checkpoint: "preserve owner history" }] }); + const [environment] = await db.insert(environments).values({ name: runId, driver: "daytona" }).returning(); + const [lease] = await db.insert(environmentLeases).values({ id: reference.leaseId, companyId, issueId: issue.id, + environmentId: environment.id, heartbeatRunId: runId, executionWorkspaceId: workspaceId, + provider: "daytona", providerLeaseId: reference.providerLeaseId, leasePolicy: "retain_on_failure", + status: "retained", metadata: { remoteCwd: reference.remoteCwd } }).returning(); + return { companyId, workspace, runId, agent, issue, reference, lease }; + } + async function retained(f: Awaited>) { + return { + runs: await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)), + owners: await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.companyId, f.companyId)), + leases: await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, f.companyId)), + issues: await db.select().from(issues).where(eq(issues.companyId, f.companyId)), + actions: await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId)), + }; + } + const inspect = (f: Awaited>) => inspectOnlineLegacyWorkspaceRecoverySource(db, f); + async function preparedWarning(f: Awaited>) { + const cause = "native_workspace_sync_out_retry_exhausted"; + const [warning] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issue.id, + kind: "active_run_watchdog", cause, fingerprint: nativeSha256({ runId: f.runId, failureCode: cause }), + ownerType: "board", returnOwnerAgentId: f.agent.id, nextAction: "Preserve the failed export", + evidence: { runId: f.runId, coordinatorAttempt: 3 } }).returning(); + return warning; + } + + it("preserves a prepared failed-export warning under its exact source and physical lock", async () => { + const f = await fixture(), recovery = await preparedWarning(f), before = await retained(f); + const held = await inspectOnlineLegacyWorkspaceRecoveryHold(db, { workspace: f.workspace, recovery }); + expect(held).toMatchObject({ finalizedHold: null, preparedSource: { + runId: f.runId, issueId: f.issue.id, reference: f.reference, + } }); + await db.transaction(async tx => { + expect(await lockOnlineLegacyWorkspaceRecoverySources(tx as unknown as Db, { + companyId: f.companyId, sources: [], holds: [held!], + })).toBe(true); + await db.transaction(async other => { + expect(await tryLockNativeWorkspaceFinalization(other as unknown as Db, f)).toBe(false); + }); + expect(await inspectOnlineLegacyWorkspaceRecoveryHold(tx, { workspace: f.workspace, recovery })).toEqual(held); + }); + expect(await retained(f)).toEqual(before); + const changed = await inspectOnlineLegacyWorkspaceRecoveryHold(db, { workspace: f.workspace, + recovery: { ...recovery, evidence: { ...recovery.evidence, coordinatorAttempt: 4 } } }); + expect(changed?.fingerprint).not.toBe(held?.fingerprint); + }); + + it.each(["run", "task", "fingerprint", "recovery_owner", "file_marker"])("rejects a prepared warning with changed %s binding", async kind => { + const f = await fixture(), recovery = await preparedWarning(f); + if (kind === "run") recovery.evidence = { runId: randomUUID() }; + if (kind === "task") { + const [child] = await db.insert(issues).values({ companyId: f.companyId, parentId: f.issue.id, title: "Related task" }).returning(); + recovery.sourceIssueId = child.id; + } + if (kind === "fingerprint") recovery.fingerprint = "00".repeat(32); + if (kind === "recovery_owner") recovery.evidence = { ...recovery.evidence, systemRecovery: { state: "running" } }; + if (kind === "file_marker") recovery.evidence = { ...recovery.evidence, workspaceRecovery: true }; + const before = await retained(f); + expect(await inspectOnlineLegacyWorkspaceRecoveryHold(db, { workspace: f.workspace, recovery })).toBeNull(); + expect(await retained(f)).toEqual(before); + }); + + it("preserves every source and historical process identity while unrelated object-backed work remains active", async () => { + const f = await fixture(); + const [other] = await db.insert(issues).values({ companyId: f.companyId, title: "Current unrelated work" }).returning(); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, nativeIssueId: other.id, + runtimeMode: "native", status: "running", controllerLeaseExpiresAt: new Date(Date.now() + 60_000), + runnerProfileJson: { remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1" } } }); + const before = await retained(f), source = await inspect(f); + expect(source).toMatchObject({ runId: f.runId, issueId: f.issue.id, reference: f.reference, + recoveryAllocations: [{ leaseId: f.lease.id, providerLeaseId: f.lease.providerLeaseId, remoteCwd: f.reference.remoteCwd }], + onlineProof: { schema: "paperclip.online-legacy-workspace-preservation.v1" } }); + await db.transaction(async tx => { + expect(await lockOnlineLegacyWorkspaceRecoverySources(tx as unknown as Db, { companyId: f.companyId, sources: [source!] })).toBe(true); + expect(await inspectOnlineLegacyWorkspaceRecoverySource(tx, f)).toEqual(source); + await db.transaction(async otherTx => { + expect(await tryLockNativeWorkspaceFinalization(otherTx as unknown as Db, f)).toBe(false); + }); + }); + expect(await retained(f)).toEqual(before); + expect(before.runs.find(run => run.id === f.runId)?.processPid).toBe(4321); + expect(before.leases[0].metadata?.remoteExecutionTermination).toBeUndefined(); + }); + + it("does not mistake the migration's own journal updates for a changed recovery source", async () => { + const f = await fixture(), source = await inspect(f); + const [workspace] = await db.update(executionWorkspaces).set({ + metadata: { remoteWorkspaceMigration: { operationId: randomUUID(), phase: "capturing" } }, updatedAt: new Date(), + }).where(eq(executionWorkspaces.id, f.workspace.id)).returning(); + expect(await inspectOnlineLegacyWorkspaceRecoverySource(db, { workspace, runId: f.runId })).toEqual(source); + }); + + it.each(["expired_coordinator", "physical_owner", "live_controller", "malformed_source", "missing_allocation"])("blocks uncertain %s without taking ownership", async change => { + const f = await fixture(); + if (change === "expired_coordinator") await db.update(nativeRunFinalizations).set({ leaseOwner: "old-owner", leaseExpiresAt: new Date(0) }).where(eq(nativeRunFinalizations.runId, f.runId)); + if (change === "physical_owner") await db.update(heartbeatRuns).set({ runnerProfileJson: { + nativeWorkspaceSync: f.reference, nativeWorkspaceFinalizationOwner: { token: "unfinished-physical-copyback" } } }).where(eq(heartbeatRuns.id, f.runId)); + if (change === "live_controller") await db.update(heartbeatRuns).set({ controllerLeaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(heartbeatRuns.id, f.runId)); + if (change === "malformed_source") await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeWorkspaceSync: { invalid: true } } }).where(eq(heartbeatRuns.id, f.runId)); + if (change === "missing_allocation") await db.update(environmentLeases).set({ providerLeaseId: randomUUID() }).where(eq(environmentLeases.id, f.lease.id)); + const before = await retained(f); + expect(await inspect(f)).toBeNull(); + expect(await retained(f)).toEqual(before); + }); + + it("keeps the source local while its physical finalizer lock is held", async () => { + const f = await fixture(), source = await inspect(f); + await db.transaction(async tx => { + expect(await tryLockNativeWorkspaceFinalization(tx as unknown as Db, f)).toBe(true); + await db.transaction(async other => { + expect(await lockOnlineLegacyWorkspaceRecoverySources(other as unknown as Db, { companyId: f.companyId, sources: [source!] })).toBe(false); + }); + }); + }); + + it("preserves all declared allocations for an ownerless result-less finalization", async () => { + const f = await fixture(); + const [second] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issue.id, + heartbeatRunId: f.runId, environmentId: f.lease.environmentId, provider: "daytona", providerLeaseId: randomUUID(), + leasePolicy: "retain_on_failure", status: "retained", metadata: { remoteCwd: "/workspaces/second" } }).returning(); + await db.update(heartbeatRuns).set({ runnerProfileJson: {}, resultJson: { checkpoint: "retain", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [second.id, f.lease.id] } } }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(nativeRunFinalizations).set({ phase: "observed" }).where(eq(nativeRunFinalizations.runId, f.runId)); + const before = await retained(f), source = await inspect(f); + expect(source?.reference).toBeNull(); + expect(source?.recoveryAllocations?.map(item => item.leaseId).sort()).toEqual([f.lease.id, second.id].sort()); + expect(source?.unresolvedFinalization).toEqual({ phase: "observed", leaseOwner: null, leaseExpiresAt: null }); + expect(await retained(f)).toEqual(before); + await db.update(environmentLeases).set({ environmentId: null }).where(eq(environmentLeases.id, second.id)); + expect(await inspect(f)).toBeNull(); + }); + + it.each(["checkpoint", "allocation", "new_owner"])("invalidates publication when %s changes after capture", async change => { + const f = await fixture(), source = await inspect(f); + await db.transaction(async tx => { + expect(await lockOnlineLegacyWorkspaceRecoverySources(tx as unknown as Db, { companyId: f.companyId, sources: [source!] })).toBe(true); + if (change === "checkpoint") await db.update(nativeRunFinalizations).set({ recoveryHistory: [{ checkpoint: "new intent" }] }).where(eq(nativeRunFinalizations.runId, f.runId)); + if (change === "allocation") await db.update(environmentLeases).set({ metadata: { remoteCwd: "/different-source" } }).where(eq(environmentLeases.id, f.lease.id)); + if (change === "new_owner") await db.update(heartbeatRuns).set({ runnerProfileJson: { + nativeWorkspaceSync: f.reference, nativeWorkspaceFinalizationOwner: { token: "uncertain owner" } } }).where(eq(heartbeatRuns.id, f.runId)); + const current = await inspectOnlineLegacyWorkspaceRecoverySource(tx, f); + expect(isDeepStrictEqual(current, source)).toBe(false); + }); + }); + + it("retains ordinary task warnings and requires a settled source for file-bearing holds", async () => { + const f = await fixture(); + const [warning] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issue.id, + kind: "configuration_validation", cause: "configuration_incomplete", fingerprint: randomUUID(), nextAction: "Keep warning" }).returning(); + const before = await retained(f); + expect(await inspectOnlineLegacyWorkspaceRecoveryHold(db, { workspace: f.workspace, recovery: warning })).toMatchObject({ finalizedHold: null }); + expect(await inspectOnlineLegacyWorkspaceRecoveryHold(db, { workspace: f.workspace, + recovery: { ...warning, evidence: { workspaceRecovery: { status: "blocked" } } } })).toBeNull(); + expect(await retained(f)).toEqual(before); + }); + + it("locks the exact finalized held run while leaving the warning and checkpoint intact", async () => { + const f = await fixture(); + await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeWorkspaceSync: { + ...f.reference, state: "finalized", finalHostSha256: "56".repeat(32) } } }).where(eq(heartbeatRuns.id, f.runId)); + const [recovery] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issue.id, + kind: "active_run_watchdog", cause: "native_continuation_requires_reconciliation", fingerprint: randomUUID(), + nextAction: "Keep the task held", evidence: { runId: f.runId, nativeFailureBlock: { retained: true } } }).returning(); + const before = await retained(f), held = await inspectOnlineLegacyWorkspaceRecoveryHold(db, { workspace: f.workspace, recovery }); + expect(held?.finalizedHold).toMatchObject({ runId: f.runId, actionId: recovery.id, sourceWorkspaceId: f.workspace.id }); + await db.transaction(async tx => { + expect(await lockOnlineLegacyWorkspaceRecoverySources(tx as unknown as Db, { + companyId: f.companyId, sources: [], holds: [held!] })).toBe(true); + await db.transaction(async other => expect(await tryLockNativeWorkspaceFinalization(other as unknown as Db, f)).toBe(false)); + }); + expect(await retained(f)).toEqual(before); + }); +}); diff --git a/server/src/services/online-legacy-workspace-recovery.ts b/server/src/services/online-legacy-workspace-recovery.ts new file mode 100644 index 00000000000..9af3f026fda --- /dev/null +++ b/server/src/services/online-legacy-workspace-recovery.ts @@ -0,0 +1,149 @@ +import { createHash } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { environmentLeases, executionWorkspaces, heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { adapterExecutionControls } from "./adapter-execution-control.js"; +import { inspectFinalizedWorkspaceRecoveryHold, type FinalizedWorkspaceRecoveryHold } from "./finalized-workspace-recovery-migration.js"; +import { isLegacyWorkspaceTaskRelated, readLegacyWorkspaceRecoveryAllocations, type LegacyWorkspaceRecoverySource } from "./legacy-workspace-recovery-migration.js"; +import { tryLockNativeWorkspaceFinalization } from "./native-runtime/native-workspace-finalization-ownership.js"; +import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js"; +import { workspaceExportCauses } from "./workspace-recovery-source.js"; +import { nativeSha256 } from "./native-runtime/canonical.js"; + +type ReadDb = Pick; +type Workspace = Pick; +type Recovery = typeof issueRecoveryActions.$inferSelect; +export const ONLINE_LEGACY_WORKSPACE_PRESERVATION = "paperclip.online-legacy-workspace-preservation.v1"; + +function canonical(value: unknown): string { + if (value instanceof Date) return JSON.stringify(value.toISOString()); + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (value && typeof value === "object") return `{${Object.entries(value).sort(([a], [b]) => a.localeCompare(b)) + .map(([key, item]) => `${JSON.stringify(key)}:${canonical(item)}`).join(",")}}`; + return JSON.stringify(value) ?? "null"; +} +const fingerprint = (value: unknown) => createHash("sha256").update(canonical(value)).digest("hex"); + +async function settledRun(db: ReadDb, workspace: Workspace, runId: string) { + const [run] = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, workspace.companyId), eq(heartbeatRuns.id, runId))); + if (!run || run.runtimeMode !== "native" || !run.nativeIssueId + || !["succeeded", "failed", "cancelled", "interrupted", "timed_out"].includes(run.status) + || run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date() + || adapterExecutionControls.has(run.id) + || run.runnerProfileJson?.nativeWorkspaceFinalizationOwner != null + || run.runnerProfileJson?.remoteWorkspace != null + || !await isLegacyWorkspaceTaskRelated(db, { companyId: workspace.companyId, + issueId: run.nativeIssueId, sourceIssueId: workspace.sourceIssueId })) return null; + const [owner] = await db.select().from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, workspace.companyId), eq(nativeRunFinalizations.runId, run.id), + eq(nativeRunFinalizations.issueId, run.nativeIssueId))); + if (!owner || owner.leaseOwner !== null) return null; + return { run, owner }; +} + +export async function inspectOnlineLegacyWorkspaceRecoverySource(db: ReadDb, input: { + workspace: Workspace; runId: string; +}): Promise { + const settled = await settledRun(db, input.workspace, input.runId); + if (!settled) return null; + const { run, owner } = settled; + const unresolved = ["observed", "retryable_failure"].includes(owner.phase) && owner.resultId === null; + if (!["committed", "terminal_failure"].includes(owner.phase) && !unresolved) return null; + const raw = run.runnerProfileJson?.nativeWorkspaceSync; + const reference = readNativeWorkspaceSyncReference(raw); + if (raw != null && !reference || reference && (reference.workspaceId !== input.workspace.id || reference.state !== "prepared")) return null; + if (!reference && !unresolved) return null; + const recoveryAllocations = await readLegacyWorkspaceRecoveryAllocations(db, run); + if (run.resultJson?.workspaceRestoreRecovery != null && !recoveryAllocations) return null; + const retainedAllocations = []; + for (const source of recoveryAllocations ?? []) { + const [lease] = await db.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, input.workspace.companyId), eq(environmentLeases.id, source.leaseId), + eq(environmentLeases.heartbeatRunId, run.id), eq(environmentLeases.issueId, run.nativeIssueId!))); + if (!lease?.environmentId || !lease.provider || lease.provider === "local" + || lease.providerLeaseId !== source.providerLeaseId || lease.metadata?.remoteCwd !== source.remoteCwd) return null; + retainedAllocations.push(lease); + } + let allocation = null; + if (reference) { + const [lease] = await db.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, input.workspace.companyId), eq(environmentLeases.id, reference.leaseId), + eq(environmentLeases.heartbeatRunId, run.id), eq(environmentLeases.issueId, run.nativeIssueId!))); + if (!lease || !lease.environmentId || !lease.provider || lease.provider === "local" + || lease.providerLeaseId !== reference.providerLeaseId || lease.metadata?.remoteCwd !== reference.remoteCwd + || lease.executionWorkspaceId && lease.executionWorkspaceId !== input.workspace.id) return null; + allocation = lease; + } + const source: LegacyWorkspaceRecoverySource = { runId: run.id, issueId: run.nativeIssueId!, reference, + ...(recoveryAllocations ? { recoveryAllocations } : {}), + ...(unresolved ? { unresolvedFinalization: { phase: owner.phase, leaseOwner: null, + leaseExpiresAt: owner.leaseExpiresAt?.toISOString() ?? null } } : {}) }; + return { ...source, onlineProof: { schema: ONLINE_LEGACY_WORKSPACE_PRESERVATION, + fingerprint: fingerprint({ workspace: { id: input.workspace.id, companyId: input.workspace.companyId, + sourceIssueId: input.workspace.sourceIssueId }, source, allocation, retainedAllocations, owner, run }) } }; +} + +export async function lockOnlineLegacyWorkspaceRecoverySources(lock: Db, input: { + companyId: string; sources: readonly LegacyWorkspaceRecoverySource[]; + holds?: readonly OnlineLegacyWorkspaceRecoveryHold[]; +}): Promise { + const runIds = new Set(); + for (const source of input.sources) { + if (source.onlineProof?.schema !== ONLINE_LEGACY_WORKSPACE_PRESERVATION + || !/^[a-f0-9]{64}$/.test(source.onlineProof.fingerprint)) return false; + runIds.add(source.runId); + } + for (const hold of input.holds ?? []) { + if (!/^[a-f0-9]{64}$/.test(hold.fingerprint)) return false; + if (hold.finalizedHold) runIds.add(hold.finalizedHold.runId); + if (hold.preparedSource) { + if (hold.preparedSource.onlineProof?.schema !== ONLINE_LEGACY_WORKSPACE_PRESERVATION + || !/^[a-f0-9]{64}$/.test(hold.preparedSource.onlineProof.fingerprint) + || hold.preparedSource.reference?.state !== "prepared") return false; + runIds.add(hold.preparedSource.runId); + } + } + for (const runId of [...runIds].sort()) { + if (!await tryLockNativeWorkspaceFinalization(lock, { companyId: input.companyId, runId })) return false; + } + return true; +} + +export type OnlineLegacyWorkspaceRecoveryHold = { + fingerprint: string; finalizedHold: FinalizedWorkspaceRecoveryHold | null; + preparedSource?: LegacyWorkspaceRecoverySource; +}; + +export async function inspectOnlineLegacyWorkspaceRecoveryHold(db: ReadDb, input: { + workspace: Workspace; recovery: Recovery; +}): Promise { + const { recovery } = input; + if (recovery.companyId !== input.workspace.companyId || !["active", "escalated"].includes(recovery.status) || !await isLegacyWorkspaceTaskRelated(db, { + companyId: input.workspace.companyId, issueId: recovery.sourceIssueId, sourceIssueId: input.workspace.sourceIssueId, + })) return null; + const fileKeys = ["workspaceRestoreRecovery", "workspaceRecovery", "historicalWorkspaceSource", "systemRecovery", "nativeFailureBlock"]; + if (["stranded_assigned_issue", "configuration_validation", "missing_disposition", "deliberate_wait_without_target"].includes(recovery.kind) + && !workspaceExportCauses.includes(recovery.cause) && !fileKeys.some(key => recovery.evidence?.[key] != null)) { + return { fingerprint: fingerprint(recovery), finalizedHold: null }; + } + if (typeof recovery.evidence?.runId !== "string") return null; + const settled = await settledRun(db, input.workspace, recovery.evidence.runId); + if (!settled) return null; + if (recovery.kind === "active_run_watchdog" && workspaceExportCauses.includes(recovery.cause) + && settled.owner.phase === "terminal_failure" && settled.owner.nextAttemptAt === null + && settled.owner.failureCode === recovery.cause && recovery.ownerType === "board" + && recovery.ownerAgentId === null && recovery.returnOwnerAgentId === settled.run.agentId && recovery.wakePolicy === null + && recovery.fingerprint === nativeSha256({ runId: settled.run.id, failureCode: recovery.cause }) + && !fileKeys.some(key => recovery.evidence?.[key] != null)) { + const source = await inspectOnlineLegacyWorkspaceRecoverySource(db, { workspace: input.workspace, runId: settled.run.id }); + if (source?.reference?.state === "prepared" && source.issueId === recovery.sourceIssueId) + return { fingerprint: fingerprint({ recovery, source }), finalizedHold: null, preparedSource: source }; + } + const held = await inspectFinalizedWorkspaceRecoveryHold(db, { + companyId: input.workspace.companyId, workspaceId: input.workspace.id, recovery, + }); + if (!held) return null; + const proof = fingerprint({ recovery, held, settled }); + return { fingerprint: proof, finalizedHold: { ...held, source: held.source ? { ...held.source, + onlineProof: { schema: ONLINE_LEGACY_WORKSPACE_PRESERVATION, fingerprint: proof } } : null } }; +} diff --git a/server/src/services/remote-workspace-migration-tree.ts b/server/src/services/remote-workspace-migration-tree.ts index c634e41b9fb..48c771763f3 100644 --- a/server/src/services/remote-workspace-migration-tree.ts +++ b/server/src/services/remote-workspace-migration-tree.ts @@ -1,8 +1,11 @@ import fs from "node:fs/promises"; import path from "node:path"; import { isDeepStrictEqual } from "node:util"; -import { and, eq, inArray, ne, sql } from "drizzle-orm"; +import { and, eq, gt, inArray, ne, sql } from "drizzle-orm"; import { executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, workspaceOperations, type Db } from "@paperclipai/db"; +import { resolvePaperclipInstanceRoot } from "../home-paths.js"; +import { adapterExecutionControls } from "./adapter-execution-control.js"; +import { managedProjectSourceIdentity } from "./workspace-source-lock.js"; import { offlineMigrationExecutionWhere } from "./offline-migration-execution.js"; import type { StorageService } from "../storage/types.js"; import { captureRemoteWorkspaceDirectory, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; @@ -46,9 +49,149 @@ export async function offlineWorkspaceMigrationBlockers(db: ReadDb) { return active ? ["active_execution_requires_drain"] : []; } +export type OnlineOrdinarySource = { + root: string; projectId: string | null; issueIds: string[]; workspaceIds: string[]; device: number; inode: number; +}; + +export async function requiresOnlineOrdinarySource(workspace: Workspace): Promise { + if (!workspace.cwd || workspace.providerType !== "local_fs" || workspace.repoUrl || workspace.branchName) return false; + const parts = path.relative(resolvePaperclipInstanceRoot(), workspace.cwd).split(path.sep); + const managed = parts[1] === workspace.companyId && Boolean(workspace.sourceIssueId) && ( + parts[0] === "projects" && parts.length === 4 && /^[a-f0-9-]{36}$/.test(parts[2]) + || parts[2] === workspace.sourceIssueId && (parts[0] === "isolated-workspaces" && parts.length === 3 + || parts[0] === "isolated-workspace-attempts" && parts.length === 4)); + if (!managed) return false; + const entry = await fs.lstat(workspace.cwd).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; throw error; + }); + return Boolean(entry?.isDirectory() && !entry.isSymbolicLink()); +} + +export async function inspectOnlineOrdinarySource(db: ReadDb, workspace: Workspace): Promise { + if (!workspace.cwd || workspace.providerType !== "local_fs" || workspace.repoUrl || workspace.branchName) return null; + const root = await physicalPath(workspace.cwd); + const instance = await physicalPath(resolvePaperclipInstanceRoot()); + const parts = path.relative(instance, root).split(path.sep); + const isolated = workspace.sourceIssueId && workspace.mode === "isolated_workspace" + && parts[1] === workspace.companyId && parts[2] === workspace.sourceIssueId + && (parts[0] === "isolated-workspaces" && parts.length === 3 + || parts[0] === "isolated-workspace-attempts" && parts.length === 4 && /^[a-zA-Z0-9_-]+$/.test(parts[3])); + const projectId = await managedProjectSourceIdentity(workspace.companyId, root); + if (root !== workspace.cwd || !(isolated || projectId && path.dirname(root) === path.join( + instance, "projects", workspace.companyId, projectId))) return null; + const entry = await fs.lstat(root).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; throw error; + }); + if (!entry?.isDirectory() || entry.isSymbolicLink()) return null; + const columns = { id: executionWorkspaces.id, companyId: executionWorkspaces.companyId, + cwd: executionWorkspaces.cwd, providerRef: executionWorkspaces.providerRef, providerType: executionWorkspaces.providerType, + repoUrl: executionWorkspaces.repoUrl, branchName: executionWorkspaces.branchName, sourceIssueId: executionWorkspaces.sourceIssueId, + strategyType: executionWorkspaces.strategyType, mode: executionWorkspaces.mode, status: executionWorkspaces.status, + metadata: sql>`jsonb_build_object( + 'gitCacheRestoration', ${executionWorkspaces.metadata}->'gitCacheRestoration', + 'nestedCloneMigrationPending', ${executionWorkspaces.metadata}->'nestedCloneMigrationPending', + 'remoteWorkspace', ${executionWorkspaces.metadata}->'remoteWorkspace')`, + }; + const rows = await db.select(columns).from(executionWorkspaces).where(and( + eq(executionWorkspaces.companyId, workspace.companyId), + inArray(executionWorkspaces.providerType, ["local_fs", "git_worktree"]))).limit(1001); + if (rows.length > 1000) return null; + const aliases: typeof rows = []; + for (const row of rows) { + const cwd = rootOf(row); + let physical: string; + try { physical = cwd ? await physicalPath(cwd) : ""; } catch { return null; } + if (!physical || !overlaps(root, physical)) continue; + if (row.cwd !== root || row.providerRef && row.providerRef !== root + || row.providerType !== "local_fs" || row.repoUrl || row.branchName || !row.sourceIssueId + || isolated && row.sourceIssueId !== workspace.sourceIssueId + || row.strategyType !== "project_primary" || !["isolated_workspace", "shared_workspace"].includes(row.mode) + || !["active", "idle", "in_review", "archived"].includes(row.status) || row.metadata?.remoteWorkspace + || row.metadata?.gitCacheRestoration || row.metadata?.nestedCloneMigrationPending) return null; + aliases.push(row); + } + let after: string | undefined; + for (let page = 0; ; page++) { + const foreign = await db.select({ id: executionWorkspaces.id, cwd: executionWorkspaces.cwd, + providerRef: executionWorkspaces.providerRef }).from(executionWorkspaces).where(and( + ne(executionWorkspaces.companyId, workspace.companyId), + inArray(executionWorkspaces.providerType, ["local_fs", "git_worktree"]), + ...(after ? [gt(executionWorkspaces.id, after)] : []), + )).orderBy(executionWorkspaces.id).limit(256); + for (const row of foreign) { + const cwd = rootOf(row); + try { if (cwd && overlaps(root, await physicalPath(cwd))) return null; } catch { return null; } + } + if (foreign.length < 256) break; + if (page === 39) return null; + after = foreign.at(-1)!.id; + } + if (!aliases.some(row => row.id === workspace.id) || aliases.length > 64) return null; + return { root, projectId, device: entry.dev, inode: entry.ino, + issueIds: [...new Set(aliases.map(row => row.sourceIssueId!))].sort(), workspaceIds: aliases.map(row => row.id).sort() }; +} + +export async function onlineOrdinarySourceBlockers(db: Pick, companyId: string, + source: OnlineOrdinarySource, workspace: Pick, operationId?: string): Promise { + const workspaceIds = sql.join(source.workspaceIds.map(id => sql`${id}::uuid`), sql`, `); + const issueIds = sql.join(source.issueIds.map(id => sql`${id}::uuid`), sql`, `); + const rows = await db.execute<{ reason: string }>(sql` + select 'active_execution' reason where exists (select 1 from heartbeat_runs r where r.company_id=${companyId}::uuid + and r.status in ('queued','running') and (r.issue_id in (${issueIds}) or r.native_issue_id in (${issueIds}) + or r.context_snapshot->>'executionWorkspaceId' in (${sql.join(source.workspaceIds.map(id => sql`${id}`), sql`, `)}) + or r.runner_profile_json#>>'{nativeWorkspaceSync,workspaceId}' in (${sql.join(source.workspaceIds.map(id => sql`${id}`), sql`, `)}))) + union all select 'task_writer_bound' where exists (select 1 from issues i where i.company_id=${companyId}::uuid + and i.id in (${issueIds}) and (i.execution_run_id is not null or i.checkout_run_id is not null)) + union all select 'workspace_operation_unresolved' where exists (select 1 from workspace_operations o where o.company_id=${companyId}::uuid + and o.execution_workspace_id in (${workspaceIds}) and (${operationId ?? null}::uuid is null or o.id<>${operationId ?? null}::uuid) + and (o.status in ('running','pending') or o.metadata->>'uncertain'='true')) + union all select 'workspace_runtime_owned' where exists (select 1 from execution_workspace_runtime_leases l where l.company_id=${companyId}::uuid + and l.execution_workspace_id in (${workspaceIds}) and l.expires_at>clock_timestamp()) + union all select 'workspace_runtime_service_unsettled' where exists (select 1 from workspace_runtime_services s where s.company_id=${companyId}::uuid + and s.status<>'stopped' and (s.execution_workspace_id in (${workspaceIds}) or s.project_workspace_id in + (select w.project_workspace_id from execution_workspaces w where w.company_id=${companyId}::uuid and w.id in (${workspaceIds})))) + union all select 'remote_copyback_unresolved' where exists (select 1 from heartbeat_runs r where r.company_id=${companyId}::uuid + and r.runner_profile_json#>>'{nativeWorkspaceSync,workspaceId}' in (${sql.join(source.workspaceIds.map(id => sql`${id}`), sql`, `)}) + and (r.runner_profile_json#>>'{nativeWorkspaceSync,workspaceId}'<>${workspace.id} + and r.runner_profile_json#>>'{nativeWorkspaceSync,state}'='prepared' + or r.runner_profile_json->'nativeWorkspaceFinalizationOwner' is not null + or r.controller_lease_expires_at>clock_timestamp())) + union all select 'execution_finalization_unresolved' where exists (select 1 from native_run_finalizations f where f.company_id=${companyId}::uuid + and (f.issue_id in (${issueIds}) and f.issue_id<>${workspace.sourceIssueId}::uuid or f.run_id in + (select r.id from heartbeat_runs r where r.company_id=${companyId}::uuid + and r.runner_profile_json#>>'{nativeWorkspaceSync,workspaceId}' in (${sql.join(source.workspaceIds.map(id => sql`${id}`), sql`, `)}) + and r.runner_profile_json#>>'{nativeWorkspaceSync,workspaceId}'<>${workspace.id})) and (f.phase not in ('committed','terminal_failure') or f.lease_owner is not null)) + union all select 'workspace_recovery_unresolved' where exists (select 1 from issue_recovery_actions a where a.company_id=${companyId}::uuid + and (a.source_issue_id in (${issueIds}) and a.source_issue_id<>${workspace.sourceIssueId}::uuid or a.source_issue_id in + (select r.native_issue_id from heartbeat_runs r where r.company_id=${companyId}::uuid + and r.runner_profile_json#>>'{nativeWorkspaceSync,workspaceId}' in (${sql.join(source.workspaceIds.map(id => sql`${id}`), sql`, `)}) + and r.runner_profile_json#>>'{nativeWorkspaceSync,workspaceId}'<>${workspace.id})) and a.status in ('active','escalated')) + union all select 'repository_preparation_unresolved' where exists (select 1 from execution_workspace_repositories r where r.company_id=${companyId}::uuid + and r.execution_workspace_id in (${workspaceIds}) and r.state<>'ready')`); + const services = await db.execute<{ cwd: string | null; provider: string }>(sql` + select cwd, provider from workspace_runtime_services where company_id=${companyId}::uuid and status<>'stopped' limit 1001`); + const reasons = rows.map(row => row.reason); + if (services.length > 1000) reasons.push("workspace_runtime_service_unsettled"); + else for (const service of services) { + try { + if (service.cwd ? overlaps(source.root, await physicalPath(service.cwd)) : service.provider !== "adapter_managed") { + reasons.push("workspace_runtime_service_unsettled"); break; + } + } catch { reasons.push("workspace_runtime_service_unsettled"); break; } + } + const controlled = [...adapterExecutionControls.keys()]; + if (controlled.length) { + const active = await db.execute(sql`select id from heartbeat_runs where company_id=${companyId}::uuid + and id in (${sql.join(controlled.map(id => sql`${id}::uuid`), sql`, `)}) + and runner_profile_json#>>'{nativeWorkspaceSync,workspaceId}' in (${sql.join(source.workspaceIds.map(id => sql`${id}`), sql`, `)}) limit 1`); + if (active.length) return [...reasons, "active_execution"]; + } + return reasons; +} + /** Only the maintenance entry point can acknowledge that every old deployment * has stopped. A task-level migration hold cannot fence other path aliases. */ -export async function workspaceMigrationLayoutBlockers(db: ReadDb, workspace: Workspace, writersStopped = false) { +export async function workspaceMigrationLayoutBlockers(db: ReadDb, workspace: Workspace, writersStopped = false, onlineSource?: OnlineOrdinarySource) { if (writersStopped) return offlineWorkspaceMigrationBlockers(db); const cwd = rootOf(workspace); if (!cwd) return []; @@ -58,7 +201,7 @@ export async function workspaceMigrationLayoutBlockers(db: ReadDb, workspace: Wo inArray(executionWorkspaces.providerType, ["local_fs", "git_worktree"]), )); const root = await physicalPath(cwd); - for (const row of local) if (rootOf(row) && overlaps(root, await physicalPath(rootOf(row)!))) + for (const row of local) if (!onlineSource?.workspaceIds.includes(row.id) && rootOf(row) && overlaps(root, await physicalPath(rootOf(row)!))) return ["overlapping_workspace_requires_maintenance"]; const migrated = await db.select({ cwd: workspaceOperations.cwd }).from(workspaceOperations).where(and( eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.phase, "remote_workspace_migration"), diff --git a/server/src/services/remote-workspace-migration.test.ts b/server/src/services/remote-workspace-migration.test.ts index 1bf6b7fcfd1..24b511bdbfd 100644 --- a/server/src/services/remote-workspace-migration.test.ts +++ b/server/src/services/remote-workspace-migration.test.ts @@ -22,7 +22,7 @@ import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archi import { captureTaskDirectorySnapshot, removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js"; import { captureRestorePoint, sealRestorePoint } from "./git-restore-point.js"; import type { TaskDirectoryRestorePointRef } from "./isolated-task-directory-retention.js"; -import { resolveManagedProjectWorkspaceDir } from "../home-paths.js"; +import { resolveManagedProjectWorkspaceDir, resolvePaperclipInstanceRoot } from "../home-paths.js"; import { cleanupExecutionWorkspaceArtifacts } from "./workspace-runtime.js"; import { workspaceOperationService } from "./workspace-operations.js"; import { resumeNativeWorkspaceSync } from "./native-runtime/native-workspace-sync.js"; @@ -965,8 +965,17 @@ describe.skipIf(!support.supported)("remote workspace migration", () => { await fs.rm(f.cwd, { recursive: true }); return ref; } - it("imports an archived directory from its encrypted restore point after the old folder is gone", async () => { - const f = await seed(), retained = await retainDirectory(f); + it.each(["historical", "canonical"] as const)("imports an archived directory from its encrypted restore point after the old folder is gone (%s)", async location => { + let f = await seed(); + if (location === "canonical") { + vi.stubEnv("PAPERCLIP_HOME", await fs.realpath(root)); + const cwd = path.join(resolvePaperclipInstanceRoot(), "isolated-workspaces", f.companyId, f.issueId); + await fs.mkdir(path.dirname(cwd), { recursive: true }); + await fs.rename(f.cwd, cwd); + await db.update(executionWorkspaces).set({ cwd, providerRef: cwd }).where(eq(executionWorkspaces.id, f.workspaceId)); + f = { ...f, cwd }; + } + const retained = await retainDirectory(f); expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: true }); await migrateRemoteWorkspace({ db, storage, ...f, apply: true }); const saved = await workspace(f.workspaceId), authority = readRemoteWorkspaceAuthority(saved.metadata)!; diff --git a/server/src/services/remote-workspace-migration.ts b/server/src/services/remote-workspace-migration.ts index 954e5a3b04f..7cd94e97e21 100644 --- a/server/src/services/remote-workspace-migration.ts +++ b/server/src/services/remote-workspace-migration.ts @@ -13,7 +13,7 @@ import { inspectFinalizedWorkspaceRecoveryHold, type FinalizedWorkspaceRecoveryH import { workspaceExportCauses } from "./workspace-recovery-source.js"; import type { StorageService } from "../storage/types.js"; import { persistActivity, publishActivity } from "./activity-log.js"; -import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js"; +import { tryAcquireExecutionWorkspaceLifecycleLock, withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js"; import { captureRemoteWorkspaceDirectory, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; import { beginWorkspaceSnapshot, loadWorkspaceSnapshotArchive, recoverWorkspaceSnapshotArchive, uploadWorkspaceSnapshot } from "./remote-workspace-snapshot-storage.js"; import { notifyRemoteWorkspaceSnapshots } from "./remote-workspace-snapshot-retention.js"; @@ -22,13 +22,16 @@ import { loadTaskDirectoryRestorePoint, mayBeIsolatedTaskDirectoryWorkspace, rea import { inspectArchivedGitMigrationSource, lockArchivedGitMigrationSources, materializeArchivedGitMigrationSource, type ArchivedGitMigrationSource } from "./remote-workspace-migration-git.js"; import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js"; import { isLegacyWorkspaceTaskRelated, readLegacyWorkspaceRecoveryAllocations, LEGACY_WORKSPACE_RECOVERY_MIGRATION, type LegacyWorkspaceRecoverySource } from "./legacy-workspace-recovery-migration.js"; -import { offlineWorkspaceMigrationBlockers, prepareNestedWorkspaceImports, validateImportedWorkspaceChildren, workspaceMigrationLayoutBlockers } from "./remote-workspace-migration-tree.js"; +import { inspectOnlineOrdinarySource, requiresOnlineOrdinarySource, onlineOrdinarySourceBlockers, type OnlineOrdinarySource, offlineWorkspaceMigrationBlockers, prepareNestedWorkspaceImports, validateImportedWorkspaceChildren, workspaceMigrationLayoutBlockers } from "./remote-workspace-migration-tree.js"; +import { withWorkspaceSourceLocks, IsolatedRepositorySourceBusy } from "./workspace-source-lock.js"; +import { inspectOnlineLegacyWorkspaceRecoverySource, inspectOnlineLegacyWorkspaceRecoveryHold, lockOnlineLegacyWorkspaceRecoverySources, type OnlineLegacyWorkspaceRecoveryHold } from "./online-legacy-workspace-recovery.js"; import { inspectNativeWorkspaceMigration, nativeWorkspaceMigrationMarker } from "./native-runtime/native-workspace-migration.js"; const SCHEMA = "paperclip.remote-workspace-migration.v1"; type Workspace = typeof executionWorkspaces.$inferSelect; type ReadDb = Pick; +type OnlineRecoveryHold = OnlineLegacyWorkspaceRecoveryHold & { actionId: string }; async function lockMigrationParents(db: ReadDb, companyId: string, issueId: string | null) { await db.select({ id: companies.id }).from(companies).where(eq(companies.id, companyId)).for("key share"); if (issueId) await db.select({ id: issues.id }).from(issues).where(and( @@ -81,15 +84,21 @@ async function isUnmaterializedRepositoryRequest(workspace: Workspace, }); } -async function inspect(db: ReadDb, input: { companyId: string; workspaceId: string; operationId?: string; writersStopped?: boolean }) { +async function inspect(db: ReadDb, input: { companyId: string; workspaceId: string; operationId?: string; writersStopped?: boolean; onlineSource?: OnlineOrdinarySource; onlineSourceUnproven?: boolean }) { const [workspace] = await db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))); if (!workspace) throw new Error("remote_workspace_migration_workspace_missing"); const authority = readRemoteWorkspaceAuthority(workspace.metadata); const marker = object(workspace.metadata?.remoteWorkspaceMigration); const allowedOperation = input.operationId ?? (marker.schema === SCHEMA && typeof marker.operationId === "string" ? marker.operationId : undefined); const reasons: string[] = []; - if (authority) return { workspace, reasons, authority, marker, source: authority.source, issueBinding: null, retainedDirectory: null, archivedGitSource: null, legacyRecoverySources: [] as LegacyWorkspaceRecoverySource[], finalizedRecoveryHolds: [] as FinalizedWorkspaceRecoveryHold[] }; + if (authority) return { workspace, reasons, authority, marker, source: authority.source, issueBinding: null, retainedDirectory: null, archivedGitSource: null, legacyRecoverySources: [] as LegacyWorkspaceRecoverySource[], finalizedRecoveryHolds: [] as FinalizedWorkspaceRecoveryHold[], onlineRecoveryHolds: [] as OnlineRecoveryHold[] }; if (input.writersStopped) reasons.push(...await offlineWorkspaceMigrationBlockers(db)); + if (input.onlineSourceUnproven) reasons.push("online_workspace_source_scope_unproven"); + if (input.onlineSource) { + const currentScope = await inspectOnlineOrdinarySource(db, workspace); + if (canonical(currentScope) !== canonical(input.onlineSource)) reasons.push("online_workspace_source_scope_changed"); + else reasons.push(...await onlineOrdinarySourceBlockers(db as Db, workspace.companyId, currentScope!, workspace, allowedOperation)); + } if (!["local_fs", "git_worktree"].includes(workspace.providerType)) reasons.push("workspace_provider_unsupported"); if (!["active", "idle", "in_review", "archived"].includes(workspace.status)) reasons.push("workspace_not_active"); if (!workspace.sourceIssueId) reasons.push("task_binding_required"); @@ -121,7 +130,7 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri ? "restore_point_expired" : "archived_git_source_unavailable"); } } if (!retainedDirectory && !archivedGitSource && (!entry?.isDirectory() || entry.isSymbolicLink())) reasons.push("local_source_unsafe_or_missing"); - if (entry?.isDirectory() && !entry.isSymbolicLink()) reasons.push(...await workspaceMigrationLayoutBlockers(db, workspace, input.writersStopped)); + if (entry?.isDirectory() && !entry.isSymbolicLink()) reasons.push(...await workspaceMigrationLayoutBlockers(db, workspace, input.writersStopped, input.onlineSource)); } const [issue] = workspace.sourceIssueId ? await db.select().from(issues).where(and(eq(issues.companyId, workspace.companyId), eq(issues.id, workspace.sourceIssueId))) : []; // Historical records still own their saved files after the task selects a @@ -147,7 +156,14 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri sql`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' in ('prepared', 'finalized')`, )); const legacyRecoverySources: LegacyWorkspaceRecoverySource[] = []; + const onlineRecoveryAllowed = Boolean(input.onlineSource) && !input.writersStopped; for (const run of referencedRuns.filter(run => object(run.reference).state === "prepared").sort((a, b) => a.id.localeCompare(b.id))) { + if (onlineRecoveryAllowed) { + const source = await inspectOnlineLegacyWorkspaceRecoverySource(db, { workspace, runId: run.id }); + if (source) legacyRecoverySources.push(source); + else reasons.push("remote_copyback_unresolved"); + continue; + } const reference = readNativeWorkspaceSyncReference(run.reference); const [owner] = await db.select({ phase: nativeRunFinalizations.phase, leaseOwner: nativeRunFinalizations.leaseOwner, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt, resultId: nativeRunFinalizations.resultId }) @@ -172,6 +188,10 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri )) : []; for (const row of finalizations) { if (legacyRecoverySources.some(source => source.runId === row.id && source.unresolvedFinalization)) continue; + if (onlineRecoveryAllowed) { + const source = await inspectOnlineLegacyWorkspaceRecoverySource(db, { workspace, runId: row.id }); + if (source) { legacyRecoverySources.push(source); continue; } + } const stopped = input.writersStopped && row.run.runtimeMode === "native" && ["succeeded", "failed", "cancelled", "interrupted", "timed_out"].includes(row.run.status) && !(row.run.controllerLeaseExpiresAt && row.run.controllerLeaseExpiresAt > new Date()) @@ -234,14 +254,26 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri if (service) reasons.push("workspace_runtime_service_unsettled"); const recoveryIssueIds = [...new Set([workspace.sourceIssueId, ...referencedRuns.map(run => run.issueId), ...legacyRecoverySources.map(source => source.issueId)].filter((id): id is string => Boolean(id)))]; - const recoveries = recoveryIssueIds.length ? await db.select({ id: issueRecoveryActions.id, sourceIssueId: issueRecoveryActions.sourceIssueId, - status: issueRecoveryActions.status, updatedAt: issueRecoveryActions.updatedAt, kind: issueRecoveryActions.kind, - cause: issueRecoveryActions.cause, evidence: issueRecoveryActions.evidence }).from(issueRecoveryActions).where(and( + const recoveries = recoveryIssueIds.length ? await db.select().from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, workspace.companyId), inArray(issueRecoveryActions.sourceIssueId, recoveryIssueIds), inArray(issueRecoveryActions.status, ["active", "escalated"]), )) : []; const finalizedRecoveryHolds: FinalizedWorkspaceRecoveryHold[] = []; + const onlineRecoveryHolds: OnlineRecoveryHold[] = []; for (const recovery of recoveries.sort((a, b) => a.id.localeCompare(b.id))) { + if (onlineRecoveryAllowed) { + const held = await inspectOnlineLegacyWorkspaceRecoveryHold(db, { workspace, recovery }); + if (!held) { reasons.push("workspace_recovery_unresolved"); continue; } + onlineRecoveryHolds.push({ actionId: recovery.id, ...held }); + if (held.preparedSource && !legacyRecoverySources.some(source => source.runId === held.preparedSource!.runId)) + legacyRecoverySources.push(held.preparedSource); + if (held.finalizedHold) { + finalizedRecoveryHolds.push(held.finalizedHold); + if (held.finalizedHold.source && !legacyRecoverySources.some(source => source.runId === held.finalizedHold!.source!.runId)) + legacyRecoverySources.push(held.finalizedHold.source); + } + continue; + } if (!input.writersStopped || object(recovery.evidence?.systemRecovery).state === "running") { reasons.push("workspace_recovery_unresolved"); continue; } @@ -272,11 +304,16 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri const materialized = repositories.filter(repository => repository.state === "ready"); if (materialized.length) source.repositories = materialized.map(repository => ({ id: repository.id, relativePath: repository.relativePath, repoUrl: repository.repoUrl, ref: repository.requestedRef, ...(repository.pinnedCommit ? { pinnedCommit: repository.pinnedCommit } : {}) })); - return { workspace, reasons: [...new Set(reasons)], authority, marker, source, issueBinding: issue?.executionWorkspaceId ?? null, retainedDirectory, archivedGitSource, legacyRecoverySources, finalizedRecoveryHolds }; + return { workspace, reasons: [...new Set(reasons)], authority, marker, source, issueBinding: issue?.executionWorkspaceId ?? null, retainedDirectory, archivedGitSource, legacyRecoverySources, finalizedRecoveryHolds, onlineRecoveryHolds }; } export async function inventoryRemoteWorkspaceMigration(db: Db, input: { companyId: string; workspaceId: string; writersStopped?: boolean }) { - const result = await inspect(db, input); + const [workspace] = !input.writersStopped ? await db.select().from(executionWorkspaces).where(and( + eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))) : []; + const onlineSource = workspace && !readRemoteWorkspaceAuthority(workspace.metadata) + ? await inspectOnlineOrdinarySource(db, workspace) ?? undefined : undefined; + const result = await inspect(db, { ...input, onlineSource, onlineSourceUnproven: Boolean(workspace && !onlineSource + && !readRemoteWorkspaceAuthority(workspace.metadata) && await requiresOnlineOrdinarySource(workspace)) }); return { companyId: input.companyId, workspaceId: input.workspaceId, eligible: result.reasons.length === 0, alreadyMigrated: Boolean(result.authority), reasons: result.reasons, localPath: result.workspace.cwd ?? result.workspace.providerRef, source: result.source, operationId: typeof result.marker.operationId === "string" ? result.marker.operationId : null, @@ -289,7 +326,16 @@ export async function migrateRemoteWorkspace(input: { writersStopped?: boolean; signal?: AbortSignal; }) { if (!input.apply) return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: false }; - return withExecutionWorkspaceLifecycleLock(input.db, input.workspaceId, async lock => { + const [sourceWorkspace] = !input.writersStopped ? await input.db.select().from(executionWorkspaces).where(and( + eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))) : []; + const onlineSource = sourceWorkspace && !readRemoteWorkspaceAuthority(sourceWorkspace.metadata) + ? await inspectOnlineOrdinarySource(input.db, sourceWorkspace) ?? undefined : undefined; + if (sourceWorkspace && !onlineSource && !readRemoteWorkspaceAuthority(sourceWorkspace.metadata) + && await requiresOnlineOrdinarySource(sourceWorkspace)) { + throw new Error("remote_workspace_migration_blocked:online_workspace_source_scope_unproven"); + } + let committedPublication: Parameters[0] | undefined; + const migrate = async (lock: Parameters[2]>[0]) => { const reserved = await input.db.transaction(async tx => { const [selected] = await tx.select({ sourceIssueId: executionWorkspaces.sourceIssueId }).from(executionWorkspaces) .where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))); @@ -298,8 +344,8 @@ export async function migrateRemoteWorkspace(input: { const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update"); if (!workspace) throw new Error("remote_workspace_migration_workspace_missing"); if (workspace.sourceIssueId !== selected.sourceIssueId) throw new Error("remote_workspace_migration_source_changed"); - const current = await inspect(tx as unknown as Db, input); - if (current.authority) return { workspace, authority: current.authority, operation: null, source: current.source, sourceFingerprint: "", issueBinding: current.issueBinding, retainedDirectory: null, archivedGitSource: null, legacyRecoverySources: [] as LegacyWorkspaceRecoverySource[], finalizedRecoveryHolds: [] as FinalizedWorkspaceRecoveryHold[] }; + const current = await inspect(tx as unknown as Db, { ...input, onlineSource }); + if (current.authority) return { workspace, authority: current.authority, operation: null, source: current.source, sourceFingerprint: "", issueBinding: current.issueBinding, retainedDirectory: null, archivedGitSource: null, legacyRecoverySources: [] as LegacyWorkspaceRecoverySource[], finalizedRecoveryHolds: [] as FinalizedWorkspaceRecoveryHold[], onlineRecoveryHolds: [] as OnlineRecoveryHold[] }; if (current.reasons.length) throw new Error(`remote_workspace_migration_blocked:${current.reasons.join(",")}`); const nativeSessionContinuity = await inspectNativeWorkspaceMigration(tx as unknown as Db, workspace); const source = remoteWorkspaceSourceSchema.parse(input.source ?? current.source); @@ -320,7 +366,8 @@ export async function migrateRemoteWorkspace(input: { if (canonical(operation.metadata?.archivedGitSource ?? null) !== canonical(gitSourceProof(current.archivedGitSource))) throw new Error("remote_workspace_migration_source_changed"); if (canonical(operation.metadata?.finalizedRecoveryHolds ?? []) !== canonical(current.finalizedRecoveryHolds)) throw new Error("remote_workspace_migration_source_changed"); if (canonical(operation.metadata?.legacyRecoverySources ?? []) !== canonical(current.legacyRecoverySources)) throw new Error("remote_workspace_migration_source_changed"); - return { workspace, authority: null, operation, source, sourceFingerprint, issueBinding, retainedDirectory: current.retainedDirectory, archivedGitSource: current.archivedGitSource, legacyRecoverySources: current.legacyRecoverySources, finalizedRecoveryHolds: current.finalizedRecoveryHolds }; + if (canonical(operation.metadata?.onlineRecoveryHolds ?? []) !== canonical(current.onlineRecoveryHolds)) throw new Error("remote_workspace_migration_source_changed"); + return { workspace, authority: null, operation, source, sourceFingerprint, issueBinding, retainedDirectory: current.retainedDirectory, archivedGitSource: current.archivedGitSource, legacyRecoverySources: current.legacyRecoverySources, finalizedRecoveryHolds: current.finalizedRecoveryHolds, onlineRecoveryHolds: current.onlineRecoveryHolds }; } const [operation] = await tx.insert(workspaceOperations).values({ companyId: workspace.companyId, executionWorkspaceId: workspace.id, issueId: workspace.sourceIssueId, cwd: workspace.cwd ?? workspace.providerRef, phase: "remote_workspace_migration", status: "running", @@ -329,6 +376,7 @@ export async function migrateRemoteWorkspace(input: { nestedWorkspaceScope: { version: 1, state: "pending" }, ...(current.finalizedRecoveryHolds.length ? { finalizedRecoveryHolds: current.finalizedRecoveryHolds } : {}), ...(current.legacyRecoverySources.length ? { legacyRecoverySources: current.legacyRecoverySources } : {}), + ...(current.onlineRecoveryHolds.length ? { onlineRecoveryHolds: current.onlineRecoveryHolds } : {}), ...(current.archivedGitSource ? { archivedGitSource: gitSourceProof(current.archivedGitSource) } : {}), ...(current.retainedDirectory ? { retainedDirectory: { restorePointId: current.retainedDirectory.restorePoint.id, generation: current.retainedDirectory.generation, digest: current.retainedDirectory.restorePoint.digest } } : {}) } }).returning(); @@ -336,10 +384,19 @@ export async function migrateRemoteWorkspace(input: { await tx.update(executionWorkspaces).set({ metadata: { ...workspace.metadata, remoteWorkspaceMigration: { schema: SCHEMA, operationId: operation!.id, state: "running", sourceFingerprint, source, } }, updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id)); - return { workspace, authority: null, operation: operation!, source, sourceFingerprint, issueBinding: current.issueBinding, retainedDirectory: current.retainedDirectory, archivedGitSource: current.archivedGitSource, legacyRecoverySources: current.legacyRecoverySources, finalizedRecoveryHolds: current.finalizedRecoveryHolds }; + return { workspace, authority: null, operation: operation!, source, sourceFingerprint, issueBinding: current.issueBinding, retainedDirectory: current.retainedDirectory, archivedGitSource: current.archivedGitSource, legacyRecoverySources: current.legacyRecoverySources, finalizedRecoveryHolds: current.finalizedRecoveryHolds, onlineRecoveryHolds: current.onlineRecoveryHolds }; }); if (reserved.authority) return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: true, replayed: true }; let operation = reserved.operation!; + if (onlineSource) { + if (!await lockOnlineLegacyWorkspaceRecoverySources(lock as unknown as Db, { companyId: input.companyId, + sources: reserved.legacyRecoverySources, holds: reserved.onlineRecoveryHolds })) + throw new Error("remote_workspace_migration_blocked:remote_copyback_owned"); + const current = await inspect(input.db, { ...input, onlineSource, operationId: operation.id }); + if (current.reasons.length || canonical(current.legacyRecoverySources) !== canonical(reserved.legacyRecoverySources) + || canonical(current.onlineRecoveryHolds) !== canonical(reserved.onlineRecoveryHolds)) + throw new Error("remote_workspace_migration_recovery_source_changed"); + } const snapshot = await beginWorkspaceSnapshot(input.db, { companyId: input.companyId, workspaceId: input.workspaceId, operationId: operation.id, generation: 1, epoch: 1 }); const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-migration-")); @@ -374,7 +431,7 @@ export async function migrateRemoteWorkspace(input: { operation = saved; } const exclude = nested.map(child => child.relativePath); - const captured = recovered ?? await captureRemoteWorkspaceDirectory({ cwd: sourceRoot!, archivePath, exclude, signal: input.signal }); + const captured = recovered ?? await captureRemoteWorkspaceDirectory({ cwd: sourceRoot!, archivePath, exclude, requireNoGit: Boolean(onlineSource), signal: input.signal }); const ref = await uploadWorkspaceSnapshot({ db: input.db, storage: input.storage, snapshotId: snapshot.id, companyId: input.companyId, workspaceId: input.workspaceId, archivePath, manifestDigest: captured.manifestDigest, signal: input.signal, // Private Git reconstruction can change administrative bytes between @@ -387,7 +444,7 @@ export async function migrateRemoteWorkspace(input: { await restoreRemoteWorkspaceArchive({ archivePath: restoredArchive, targetDir: restored, expectedManifestDigest: ref.manifestDigest, signal: input.signal }); if (!recovered) { const currentArchive = path.join(scratch, "current.pcws"); - const current = await captureRemoteWorkspaceDirectory({ cwd: sourceRoot!, archivePath: currentArchive, exclude, signal: input.signal }); + const current = await captureRemoteWorkspaceDirectory({ cwd: sourceRoot!, archivePath: currentArchive, exclude, requireNoGit: Boolean(onlineSource), signal: input.signal }); if (current.manifestDigest !== captured.manifestDigest) throw new Error("remote_workspace_migration_source_changed"); } if (nested.length) { @@ -397,11 +454,12 @@ export async function migrateRemoteWorkspace(input: { } input.signal?.throwIfAborted(); await lock.execute(sql`select 1`); - const publication = await input.db.transaction(async tx => { + const publicationDb = onlineSource ? lock as unknown as Db : input.db; + const publication = await publicationDb.transaction(async tx => { await lockMigrationParents(tx, input.companyId, reserved.workspace.sourceIssueId); - for (const held of reserved.finalizedRecoveryHolds) { + for (const actionId of [...new Set([...reserved.finalizedRecoveryHolds, ...reserved.onlineRecoveryHolds].map(held => held.actionId))].sort()) { await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( - eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.id, held.actionId))).for("update", { noWait: true }); + eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.id, actionId))).for("update", { noWait: true }); } const protectedRunIds = [...new Set([...reserved.legacyRecoverySources.map(source => source.runId), ...reserved.finalizedRecoveryHolds.map(held => held.runId)])].sort(); @@ -422,7 +480,7 @@ export async function migrateRemoteWorkspace(input: { .from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.id, allocation.leaseId))).for("update", { noWait: true }); } - const validation = await inspect(tx as unknown as Db, { ...input, operationId: operation.id }); + const validation = await inspect(tx as unknown as Db, { ...input, operationId: operation.id, onlineSource }); if (!input.source && canonical(validation.source) !== canonical(reserved.source)) throw new Error("remote_workspace_migration_source_changed"); if (validation.issueBinding !== reserved.issueBinding) throw new Error("remote_workspace_migration_source_changed"); if (Boolean(validation.retainedDirectory) !== Boolean(reserved.retainedDirectory)) throw new Error("remote_workspace_migration_source_changed"); @@ -441,6 +499,7 @@ export async function migrateRemoteWorkspace(input: { || saved.sourceOperationId !== child.operationId) throw new Error("nested_workspace_snapshot_changed"); } if (canonical(validation.finalizedRecoveryHolds) !== canonical(reserved.finalizedRecoveryHolds)) throw new Error("remote_workspace_migration_source_changed"); + if (canonical(validation.onlineRecoveryHolds) !== canonical(reserved.onlineRecoveryHolds)) throw new Error("remote_workspace_migration_recovery_source_changed"); if (canonical(validation.legacyRecoverySources) !== canonical(reserved.legacyRecoverySources)) throw new Error("remote_workspace_migration_source_changed"); const continuity = await inspectNativeWorkspaceMigration(tx as unknown as Db, workspace); if (canonical(continuity) !== canonical(operation.metadata?.nativeSessionContinuity ?? null)) throw new Error("native_workspace_migration_source_changed"); @@ -492,8 +551,26 @@ export async function migrateRemoteWorkspace(input: { if ((cause.code ?? failure.code) !== "55P03") throw error; throw new Error("remote_workspace_migration_blocked:execution_rows_busy", { cause: error }); }); - if (publication) publishActivity(publication); - return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: true, replayed: false }; + if (publication) { + if (onlineSource) committedPublication = publication; + else publishActivity(publication); + } + return { ...await inventoryRemoteWorkspaceMigration(publicationDb, input), applied: true, replayed: false }; } finally { await removeTaskDirectoryTree(scratch); } - }); + }; + if (onlineSource) { + const result = await withWorkspaceSourceLocks(input.db, { + companyId: input.companyId, issueIds: onlineSource.issueIds, + projectIds: onlineSource.projectId ? [onlineSource.projectId] : [], + }, async lock => { + for (const workspaceId of onlineSource.workspaceIds) { + if (!await tryAcquireExecutionWorkspaceLifecycleLock(lock, workspaceId)) + throw new IsolatedRepositorySourceBusy("workspace_source_busy"); + } + return migrate(lock); + }, true); + if (committedPublication) publishActivity(committedPublication); + return result; + } + return withExecutionWorkspaceLifecycleLock(input.db, input.workspaceId, migrate); } diff --git a/server/src/services/remote-workspace-online-source.test.ts b/server/src/services/remote-workspace-online-source.test.ts new file mode 100644 index 00000000000..9f9e0c5cc2f --- /dev/null +++ b/server/src/services/remote-workspace-online-source.test.ts @@ -0,0 +1,401 @@ +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { sql, eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { activityLog, agents, companies, createDb, environmentLeases, environments, executionWorkspaceSnapshots, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices, type Db } from "@paperclipai/db"; +import { readRemoteWorkspaceAuthority, readRemoteWorkspaceRunReference } from "@paperclipai/shared"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { resolveManagedProjectWorkspaceDir, resolvePaperclipInstanceRoot } from "../home-paths.js"; +import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; +import { createStorageService } from "../storage/service.js"; +import type { StorageService } from "../storage/types.js"; +import { inventoryRemoteWorkspaceMigration, migrateRemoteWorkspace } from "./remote-workspace-migration.js"; +import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "./remote-workspace-snapshot-storage.js"; +import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; +import { materializeManagedProjectWorkspace } from "./managed-repository-checkout.js"; +import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js"; +import { IsolatedRepositorySourceBusy, withIsolatedRepositorySourceLock, withWorkspaceSourceLocks } from "./workspace-source-lock.js"; +import { tryLockNativeWorkspaceFinalization } from "./native-runtime/native-workspace-finalization-ownership.js"; +import { recordNativeFinalizationFailure } from "./native-runtime/native-run-finalizer.js"; +import { withWorkspaceMoveLocks } from "./isolated-repository-rehome.js"; +import { ensureRuntimeServicesForRun, startRuntimeServicesForWorkspaceControl, stopRuntimeServicesForExecutionWorkspace } from "./workspace-runtime.js"; + +const support = await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("online ordinary physical source fence", () => { + let database: Awaited>; + let db: ReturnType, home: string, storage: StorageService; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("online-ordinary-source-"); + db = createDb(database.connectionString, { maxConnections: 1 }); + await db.execute(sql`set client_min_messages = warning`); + }, 90_000); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + beforeEach(async () => { + home = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "pc-online-source-"))); + vi.stubEnv("PAPERCLIP_HOME", home); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "61".repeat(32)); + storage = createStorageService(createLocalDiskStorageProvider(path.join(home, "objects"))); + }); + afterEach(async () => { + vi.restoreAllMocks(); + await db.delete(activityLog); + await db.delete(workspaceOperations); + await db.delete(executionWorkspaceSnapshots); + await db.delete(workspaceRuntimeServices); + await db.delete(nativeRunFinalizations); + await db.delete(environmentLeases); + await db.delete(environments); + await db.delete(heartbeatRuns); + await db.delete(issues); + await db.delete(agents); + await db.delete(executionWorkspaces); + await db.delete(projectWorkspaces); + await db.delete(projects); + await db.delete(companies); + vi.unstubAllEnvs(); + await fs.rm(home, { recursive: true, force: true }); + }); + async function fixture() { + const companyId = randomUUID(), projectId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Online source", issuePrefix: `O${companyId.slice(0, 6)}` }); + await db.insert(projects).values({ id: projectId, companyId, name: "Shared legacy project" }); + const cwd = resolveManagedProjectWorkspaceDir({ companyId, projectId }); + await fs.mkdir(cwd, { recursive: true }); + await fs.writeFile(path.join(cwd, "saved.txt"), "saved legacy edits\n"); + const aliases = []; + for (let n = 0; n < 3; n++) { + const [issue] = await db.insert(issues).values({ companyId, projectId, title: `Legacy task ${n}` }).returning(); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, projectId, sourceIssueId: issue.id, + name: `Alias ${n}`, mode: "shared_workspace", strategyType: "project_primary", providerType: "local_fs", cwd, providerRef: cwd }).returning(); + await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issue.id)); + aliases.push({ issue, workspace }); + } + return { companyId, projectId, cwd, aliases, workspaceId: aliases[0].workspace.id }; + } + async function stored(workspaceId: string) { + return (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId)))[0]; + } + function uploadBarrier() { + let entered!: () => void, release!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + const pending = new Promise(resolve => { release = resolve; }); + const put = storage.putFile.bind(storage); + vi.spyOn(storage, "putFile").mockImplementationOnce(async input => { + const result = await put(input); entered(); await pending; return result; + }); + return { started, release }; + } + async function remoteWriter(companyId: string, agentId: string) { + const workspaceId = randomUUID(), runId = randomUUID(), leaseId = randomUUID(), providerLeaseId = randomUUID(); + const [issue] = await db.insert(issues).values({ companyId, title: "Unrelated object-backed task" }).returning(); + const [environment] = await db.insert(environments).values({ name: runId, driver: "daytona" }).returning(); + const remoteCwd = `/workspaces/${workspaceId}`; + const authority = { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null, source: { kind: "empty" }, + location: { environmentId: environment.id, leaseId, providerLeaseId, remoteCwd }, writerRunId: runId, dirty: true }; + const [workspace] = await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issue.id, + name: "Unrelated remote writer", mode: "isolated_workspace", strategyType: "project_primary", providerType: "cloud_sandbox", + cwd: remoteCwd, metadata: { remoteWorkspace: authority } }).returning(); + const reference = { schema: "paperclip.remote-workspace-run.v1", workspaceId, leaseId, providerLeaseId, + remoteCwd, epoch: 1, state: "prepared", snapshotId: null }; + const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId: issue.id, + nativeIssueId: issue.id, status: "running", runtimeMode: "native", contextSnapshot: { executionWorkspaceId: workspaceId }, + runnerProfileJson: { remoteWorkspace: reference } }).returning(); + await db.insert(environmentLeases).values({ id: leaseId, companyId, heartbeatRunId: runId, issueId: issue.id, + executionWorkspaceId: workspaceId, environmentId: environment.id, provider: "daytona", providerLeaseId, + status: "active", metadata: { remoteCwd } }); + expect(readRemoteWorkspaceAuthority(workspace.metadata)).toEqual(authority); + expect(readRemoteWorkspaceRunReference(run.runnerProfileJson?.remoteWorkspace)).toEqual(reference); + return run; + } + it("imports an alias with unrelated object-backed execution still running and leaves all original files", async () => { + const f = await fixture(); + const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Unrelated active agent" }).returning(); + const run = await remoteWriter(f.companyId, agent.id); + expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: true }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).resolves.toMatchObject({ applied: true }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("saved legacy edits\n"); + expect(await stored(f.aliases[1].workspace.id)).toEqual(f.aliases[1].workspace); + const authority = readRemoteWorkspaceAuthority((await stored(f.workspaceId)).metadata)!; + const reference = await getWorkspaceSnapshot(db, { ...f, snapshotId: authority.snapshotId! }); + const archivePath = path.join(home, "restored.pcws"), targetDir = path.join(home, "restored"); + await loadWorkspaceSnapshotArchive({ storage, reference: reference!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: reference!.manifestDigest }); + expect(await fs.readFile(path.join(targetDir, "saved.txt"), "utf8")).toBe("saved legacy edits\n"); + }); + it("fences future issues and alias lifecycle owners while another project can materialize", async () => { + const f = await fixture(), barrier = uploadBarrier(); + const migration = migrateRemoteWorkspace({ db, storage, ...f, apply: true }); + await Promise.race([barrier.started, migration.then(() => { throw new Error("capture did not reach upload"); })]); + let lifecycleEntered = false, preparationEntered = false; + const preparation = withIsolatedRepositorySourceLock(db, f.companyId, randomUUID(), async () => { + await materializeManagedProjectWorkspace(path.join(f.cwd, "new-issue"), { repoUrl: null }); + preparationEntered = true; + }); + const lifecycle = withExecutionWorkspaceLifecycleLock(db, f.aliases[1].workspace.id, async () => { lifecycleEntered = true; }); + try { + expect(await fs.lstat(path.join(f.cwd, "new-issue")).catch(() => null)).toBeNull(); + const unrelated = resolveManagedProjectWorkspaceDir({ companyId: f.companyId, projectId: randomUUID() }); + await withIsolatedRepositorySourceLock(db, f.companyId, randomUUID(), () => + materializeManagedProjectWorkspace(unrelated, { repoUrl: null })); + expect((await fs.stat(unrelated)).isDirectory()).toBe(true); + await vi.waitFor(async () => { + const rows = await db.execute(sql`select count(*)::int n from pg_stat_activity where datname=current_database() and wait_event='advisory'`); + expect(rows[0].n).toBeGreaterThan(0); + }); + expect(lifecycleEntered).toBe(false); + expect(preparationEntered).toBe(false); + } finally { barrier.release(); await migration; await lifecycle; await preparation; } + expect(lifecycleEntered).toBe(true); + expect(preparationEntered).toBe(true); + }); + it("checks every alias issue lock before creating a migration journal", async () => { + const f = await fixture(); + let entered!: () => void, release!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + const pending = new Promise(resolve => { release = resolve; }); + const owner = withIsolatedRepositorySourceLock(db, f.companyId, f.aliases[2].issue.id, async () => { entered(); await pending; }); + await started; + try { + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toBeInstanceOf(IsolatedRepositorySourceBusy); + expect(await db.select().from(workspaceOperations)).toEqual([]); + } finally { release(); await owner; } + }); + it("fences the actual rehome and nested-conversion owner through its project source", async () => { + const f = await fixture(); + let entered!: () => void, release!: () => void, moved = false; + const started = new Promise(resolve => { entered = resolve; }); + const pending = new Promise(resolve => { release = resolve; }); + const owner = withWorkspaceSourceLocks(db, { companyId: f.companyId, projectIds: [f.projectId] }, async () => { entered(); await pending; }); + await started; + const movement = withWorkspaceMoveLocks(db, f.aliases[0].workspace, async () => { moved = true; }); + try { + await vi.waitFor(async () => { + const rows = await db.execute(sql`select count(*)::int n from pg_stat_activity where datname=current_database() and wait_event='advisory'`); + expect(rows[0].n).toBeGreaterThan(0); + }); + expect(moved).toBe(false); + } finally { release(); await owner; await movement; } + expect(moved).toBe(true); + }); + it("keeps runs without runtime services free of source filesystem admission", async () => { + const f = await fixture(), realpath = fs.realpath.bind(fs); + vi.spyOn(fs, "realpath").mockImplementation(async target => { + if (String(target).startsWith(home)) throw new Error("runtime sources must stay unused"); + return realpath(target); + }); + expect(await ensureRuntimeServicesForRun({ db, runId: randomUUID(), executionWorkspaceId: f.workspaceId, + agent: { id: null, name: "No service owner", companyId: f.companyId }, issue: null, adapterEnv: {}, config: {}, skipHostProvision: true, + workspace: { cwd: f.cwd, baseCwd: f.cwd, source: "project_primary", projectId: f.projectId, workspaceId: null, + repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, branchCreatedByRuntime: false, + worktreePath: null, warnings: [], created: false } })).toEqual([]); + }); + it.each(["workspace", "configured"] as const)("fences project command provisioning and its running %s CWD without an execution workspace ID", async location => { + const f = await fixture(); + const startProjectId = location === "configured" ? randomUUID() : f.projectId; + if (startProjectId !== f.projectId) await db.insert(projects).values({ id: startProjectId, companyId: f.companyId, name: "Service owner" }); + const startCwd = resolveManagedProjectWorkspaceDir({ companyId: f.companyId, projectId: startProjectId }); + await fs.mkdir(startCwd, { recursive: true }); + const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Project command owner" }).returning(); + const [projectWorkspace] = await db.insert(projectWorkspaces).values({ companyId: f.companyId, projectId: startProjectId, + name: "Project source", cwd: startCwd }).returning(); + let entered!: () => void, release!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + const pending = new Promise(resolve => { release = resolve; }); + const owner = withWorkspaceSourceLocks(db, { companyId: f.companyId, projectIds: [f.projectId] }, async () => { entered(); await pending; }); + await started; + const node = JSON.stringify(process.execPath), marker = path.join(startCwd, "runtime-source-marker"); + const services = startRuntimeServicesForWorkspaceControl({ db, actor: agent, issue: null, adapterEnv: {}, + workspace: { cwd: startCwd, baseCwd: startCwd, source: "project_primary", projectId: startProjectId, workspaceId: projectWorkspace.id, + repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, branchCreatedByRuntime: false, + worktreePath: null, warnings: [], created: false }, + config: { runtimeProvisionCommand: `${node} -e ${JSON.stringify("require('node:fs').writeFileSync('runtime-source-marker', 'provisioned')")}`, + workspaceRuntime: { services: [{ name: "source-fence-proof", cwd: f.cwd, command: `${node} -e ${JSON.stringify("setInterval(() => {}, 1000)")}`, + lifecycle: "shared", reuseScope: "project_workspace", stopPolicy: { type: "manual" } }] } }, + }); + try { + await vi.waitFor(async () => { + const rows = await db.execute(sql`select count(*)::int n from pg_stat_activity where datname=current_database() and wait_event='advisory'`); + expect(rows[0].n).toBeGreaterThan(0); + }); + expect(await fs.lstat(marker).catch(() => null)).toBeNull(); + } finally { + release(); await owner; + const refs = await services; + try { + expect(refs).toHaveLength(1); + expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: false, + reasons: expect.arrayContaining(["workspace_runtime_service_unsettled"]) }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("workspace_runtime_service_unsettled"); + expect(readRemoteWorkspaceAuthority((await stored(f.workspaceId)).metadata)).toBeNull(); + } finally { + for (const ref of refs) await stopRuntimeServicesForExecutionWorkspace({ db, executionWorkspaceId: f.workspaceId, runtimeServiceId: ref.id }); + } + } + expect(await fs.readFile(marker, "utf8")).toBe("provisioned"); + expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: true, reasons: [] }); + }); + it("fences a configured alias of an existing isolated task directory using its original issue key", async () => { + const f = await fixture(), sourceIssueId = f.aliases[0].issue.id; + const cwd = path.join(resolvePaperclipInstanceRoot(), "isolated-workspaces", f.companyId, sourceIssueId); + await fs.mkdir(cwd, { recursive: true }); + let entered!: () => void, release!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + const pending = new Promise(resolve => { release = resolve; }); + const owner = withIsolatedRepositorySourceLock(db, f.companyId, sourceIssueId, async () => { entered(); await pending; }); + await started; + try { + await expect(withIsolatedRepositorySourceLock(db, f.companyId, randomUUID(), () => + materializeManagedProjectWorkspace(path.join(cwd, "new-alias"), { repoUrl: null }))) + .rejects.toBeInstanceOf(IsolatedRepositorySourceBusy); + expect(await fs.lstat(path.join(cwd, "new-alias")).catch(() => null)).toBeNull(); + } finally { release(); await owner; } + }); + it("bounds source-lock closures before reserving a connection", async () => { + await expect(withWorkspaceSourceLocks(db, { companyId: randomUUID(), issueIds: Array.from({ length: 129 }, () => randomUUID()) }, + async () => { throw new Error("scope must not be admitted"); })).rejects.toThrow("workspace_source_scope_limit"); + }); + it.each(["task", "operation", "foreign", "descendant"] as const)("retains the %s guard across the complete alias closure", async kind => { + const f = await fixture(), alias = f.aliases[2]; + if (kind === "task") { + const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Alias checkout" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent.id, status: "succeeded" }).returning(); + await db.update(issues).set({ checkoutRunId: run.id }).where(eq(issues.id, alias.issue.id)); + } + if (kind === "operation") await db.insert(workspaceOperations).values({ companyId: f.companyId, + executionWorkspaceId: alias.workspace.id, phase: "cleanup", status: "running" }); + if (kind === "foreign") { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Foreign", issuePrefix: "FOREIGN" }); + await db.insert(executionWorkspaces).values({ companyId, name: "Foreign alias", mode: "shared_workspace", + strategyType: "project_primary", providerType: "local_fs", cwd: f.cwd }); + } + if (kind === "descendant") await db.update(executionWorkspaces).set({ cwd: path.join(f.cwd, "child") }).where(eq(executionWorkspaces.id, alias.workspace.id)); + expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: false }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("remote_workspace_migration_blocked"); + expect(readRemoteWorkspaceAuthority((await stored(f.workspaceId)).metadata)).toBeNull(); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("saved legacy edits\n"); + }); + it("does not count unrelated tenants against the company alias bound", async () => { + const f = await fixture(), companyId = randomUUID(), cwd = path.join(home, "other-tenant"); + await fs.mkdir(cwd); + await db.insert(companies).values({ id: companyId, name: "Unrelated tenant", issuePrefix: "OTHER" }); + await db.insert(executionWorkspaces).values(Array.from({ length: 1001 }, () => ({ + companyId, name: "Unrelated workspace", mode: "shared_workspace", strategyType: "project_primary", + providerType: "local_fs", cwd, + }))); + expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: true, reasons: [] }); + }); + it.each(["symlink", "unreadable"] as const)("blocks an unproven foreign %s path even for a single local alias", async kind => { + const f = await fixture(), companyId = randomUUID(), cwd = path.join(home, "foreign-alias"); + for (const alias of f.aliases.slice(1)) await db.update(executionWorkspaces).set({ providerType: "plugin" }) + .where(eq(executionWorkspaces.id, alias.workspace.id)); + await db.insert(companies).values({ id: companyId, name: "Foreign alias", issuePrefix: "FOREIGN" }); + if (kind === "symlink") await fs.symlink(f.cwd, cwd); + else { + const realpath = fs.realpath.bind(fs); + vi.spyOn(fs, "realpath").mockImplementation(async value => { + if (value === cwd) throw Object.assign(new Error("fixture path inaccessible"), { code: "EACCES" }); + return realpath(value); + }); + } + await db.insert(executionWorkspaces).values({ companyId, name: "Foreign path", mode: "shared_workspace", + strategyType: "project_primary", providerType: "local_fs", cwd }); + expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: false, + reasons: expect.arrayContaining(["online_workspace_source_scope_unproven"]) }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("online_workspace_source_scope_unproven"); + expect(await db.select().from(workspaceOperations)).toEqual([]); + expect(readRemoteWorkspaceAuthority((await stored(f.workspaceId)).metadata)).toBeNull(); + }); + it("rechecks closure membership before publication and preserves the uploaded journal on change", async () => { + const f = await fixture(), put = storage.putFile.bind(storage); + vi.spyOn(storage, "putFile").mockImplementationOnce(async input => { + const result = await put(input); + await db.update(executionWorkspaces).set({ providerRef: "/unproven/new/source" }).where(eq(executionWorkspaces.id, f.aliases[2].workspace.id)); + return result; + }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("online_workspace_source_scope_changed"); + expect(readRemoteWorkspaceAuthority((await stored(f.workspaceId)).metadata)).toBeNull(); + expect((await db.select().from(workspaceOperations))[0].status).toBe("running"); + }); + it("does not publish after the actual source lock connection is terminated", async () => { + const f = await fixture(), barrier = uploadBarrier(); + const migration = migrateRemoteWorkspace({ db, storage, ...f, apply: true }).then(value => ({ value }), error => ({ error })); + await Promise.race([barrier.started, migration.then(result => { if ("error" in result) throw result.error; throw new Error("capture did not reach upload"); })]); + const key = `managed_project_source:${f.companyId}:${f.projectId}`; + const rows = await db.execute(sql`select distinct a.pid from pg_stat_activity a join pg_locks l on l.pid=a.pid + where a.datname=current_database() and l.locktype='advisory' and l.granted + and l.classid=((hashtextextended(${key},0)>>32)&4294967295)::oid + and l.objid=(hashtextextended(${key},0)&4294967295)::oid`); + expect(rows).toHaveLength(1); + await db.execute(sql`select pg_terminate_backend(${Number(rows[0].pid)})`); + barrier.release(); + expect(await migration).toHaveProperty("error"); + expect(readRemoteWorkspaceAuthority((await stored(f.workspaceId)).metadata)).toBeNull(); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("saved legacy edits\n"); + }); + async function recoveryFixture() { + const f = await fixture(), runId = randomUUID(); + const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Terminal native worker", adapterType: "paperclip_runner" }).returning(); + const reference = { schema: "paperclip.native-workspace-sync/v3", state: "prepared", workspaceId: f.workspaceId, + leaseId: randomUUID(), providerLeaseId: randomUUID(), remoteCwd: "/workspaces/retained", + descriptorSha256: "12".repeat(32), baselineSha256: "34".repeat(32), resourceDisposition: "destroy" }; + await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: agent.id, + nativeIssueId: f.aliases[0].issue.id, status: "failed", runtimeMode: "native", runnerProfileJson: { nativeWorkspaceSync: reference } }).returning(); + await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.aliases[0].issue.id, + runId, phase: "observed" }); + const [environment] = await db.insert(environments).values({ name: runId, driver: "daytona" }).returning(); + const [lease] = await db.insert(environmentLeases).values({ id: reference.leaseId, companyId: f.companyId, + issueId: f.aliases[0].issue.id, heartbeatRunId: runId, executionWorkspaceId: f.workspaceId, environmentId: environment.id, + provider: "daytona", providerLeaseId: reference.providerLeaseId, status: "retained", leasePolicy: "retain_on_failure", + metadata: { remoteCwd: reference.remoteCwd } }).returning(); + await recordNativeFinalizationFailure({ db, runId, error: new Error("native_workspace_sync_out_unrecoverable"), + failureScope: "workspace", permanent: true }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const [owner] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + const [warning] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.aliases[0].issue.id)); + expect(warning).toMatchObject({ kind: "active_run_watchdog", cause: "native_workspace_sync_out_unrecoverable", + status: "active", evidence: { runId } }); + return { ...f, run, owner, lease, warning }; + } + it("imports a terminal prepared source while retaining its original provider and finalization obligations", async () => { + const f = await recoveryFixture(), put = storage.putFile.bind(storage); + const unrelated = await remoteWriter(f.companyId, f.run.agentId); + expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: true }); + vi.spyOn(storage, "putFile").mockImplementationOnce(async input => { + await db.transaction(async tx => { + expect(await tryLockNativeWorkspaceFinalization(tx as unknown as Db, { companyId: f.companyId, runId: f.run.id })).toBe(false); + }); + return put(input); + }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).resolves.toMatchObject({ applied: true }); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.run.id)))[0]).toEqual(f.owner); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0]).toEqual(f.lease); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.warning.id)))[0]).toEqual(f.warning); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, unrelated.id)))[0]).toEqual(unrelated); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)); + expect(run.runnerProfileJson?.nativeWorkspaceSync).toEqual(f.run.runnerProfileJson?.nativeWorkspaceSync); + expect(run.runnerProfileJson?.legacyWorkspaceRecoveryMigration).toMatchObject({ workspaceId: f.workspaceId }); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("saved legacy edits\n"); + }); + it.each(["expired_owner", "checkpoint", "allocation"] as const)("keeps a prepared source local when %s changes", async change => { + const f = await recoveryFixture(); + if (change === "expired_owner") await db.update(nativeRunFinalizations).set({ leaseOwner: "still-owned", leaseExpiresAt: new Date(0) }) + .where(eq(nativeRunFinalizations.runId, f.run.id)); + else { + const put = storage.putFile.bind(storage); + vi.spyOn(storage, "putFile").mockImplementationOnce(async input => { + const result = await put(input); + if (change === "checkpoint") await db.update(nativeRunFinalizations).set({ recoveryHistory: [{ intent: "changed" }] }) + .where(eq(nativeRunFinalizations.runId, f.run.id)); + else await db.update(environmentLeases).set({ metadata: { remoteCwd: "/changed-allocation" } }).where(eq(environmentLeases.id, f.lease.id)); + return result; + }); + } + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow(); + expect(readRemoteWorkspaceAuthority((await stored(f.workspaceId)).metadata)).toBeNull(); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("saved legacy edits\n"); + }); +}); diff --git a/server/src/services/remote-workspace-snapshot-archive.test.ts b/server/src/services/remote-workspace-snapshot-archive.test.ts index d888be0cd36..b7f166ffec5 100644 --- a/server/src/services/remote-workspace-snapshot-archive.test.ts +++ b/server/src/services/remote-workspace-snapshot-archive.test.ts @@ -68,6 +68,44 @@ describe("portable remote workspace archives", () => { const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; } + it.each([".", "nested"])("requires a shared Git fence before reading included %s Git storage", async location => { + const cwd = await source(), repository = path.join(cwd, location); + await fs.mkdir(repository, { recursive: true }); + await git(repository, "init", "--quiet"); + await fs.writeFile(path.join(repository, "saved.txt"), "preserve original work\n"); + await git(repository, "add", "."); + await git(repository, "commit", "-qm", "Saved work"); + const before = await sourceBytes(cwd); + await expect(captureRemoteWorkspaceDirectory({ cwd, archivePath: path.join(root, "fenced.pcws"), requireNoGit: true })) + .rejects.toThrow("online_source_git_requires_shared_fence"); + expect(await sourceBytes(cwd)).toBe(before); + await expect(capture(cwd)).resolves.toMatchObject({ manifestDigest: expect.any(String) }); + }); + it("requires a shared Git fence for a nested bare repository borrowing external objects", async () => { + const cwd = await source(), upstream = path.join(root, "upstream"), repository = path.join(cwd, "bare.git"); + await fs.mkdir(upstream); + await git(upstream, "init", "--quiet"); + await fs.writeFile(path.join(upstream, "saved.txt"), "external commit objects\n"); + await git(upstream, "add", "."); + await git(upstream, "commit", "-qm", "External source"); + await git(cwd, "clone", "--bare", "--shared", upstream, repository); + expect(await fs.readFile(path.join(repository, "objects", "info", "alternates"), "utf8")) + .toContain(path.join(upstream, ".git", "objects")); + const before = await sourceBytes(cwd), external = await sourceBytes(upstream); + await expect(captureRemoteWorkspaceDirectory({ cwd, archivePath: path.join(root, "bare.pcws"), requireNoGit: true })) + .rejects.toThrow("online_source_git_requires_shared_fence"); + expect(await sourceBytes(cwd)).toBe(before); + expect(await sourceBytes(upstream)).toBe(external); + }); + it.each(["commondir", "alternates"])("requires a shared Git fence for included detached %s metadata", async marker => { + const cwd = await source(), directory = path.join(cwd, marker === "commondir" ? "admin" : "object-cache/info"); + await fs.mkdir(directory, { recursive: true }); + await fs.writeFile(path.join(directory, marker), "/outside/source/Git-storage\n"); + const before = await sourceBytes(cwd); + await expect(captureRemoteWorkspaceDirectory({ cwd, archivePath: path.join(root, `${marker}.pcws`), requireNoGit: true })) + .rejects.toThrow("online_source_git_requires_shared_fence"); + expect(await sourceBytes(cwd)).toBe(before); + }); async function parallelSource() { const cwd = await source(); for (const name of ["a", "b", "c", "d", "e", "f"]) { diff --git a/server/src/services/remote-workspace-snapshot-archive.ts b/server/src/services/remote-workspace-snapshot-archive.ts index d4928e8bb0a..f554e793720 100644 --- a/server/src/services/remote-workspace-snapshot-archive.ts +++ b/server/src/services/remote-workspace-snapshot-archive.ts @@ -10,6 +10,7 @@ async function workspaceArchive(input: { targetDir?: string; expectedManifestDigest?: string; exclude?: string[]; + requireNoGit?: boolean; signal?: AbortSignal; maxEntries?: number; maxManifestBytes?: number; @@ -324,8 +325,15 @@ async function workspaceArchive(input: { const directoryStat = await fs.lstat(directory); if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink() || await fs.realpath(directory) !== directory) fail("changed"); const names = (await fs.readdir(directory)).sort(); + if (input.requireNoGit && (names.includes("commondir") || names.includes("gitdir") + || names.includes("objects") && names.some(name => ["HEAD", "refs", "packed-refs"].includes(name)) + || names.includes("HEAD") && names.includes("config") + || path.basename(directory) === "info" && names.some(name => ["alternates", "http-alternates"].includes(name)))) { + fail("online_source_git_requires_shared_fence"); + } const inRepository = parentIsRepository || names.includes(".git"); if (skipGit && names.includes(".git") && !excluded(relative ? `${relative}/.git` : ".git")) { + if (input.requireNoGit) fail("online_source_git_requires_shared_fence"); // Validate the repository before any Git read below, including the // tracked check for node_modules, can use a damaged entry. if ((await fs.lstat(path.join(directory, ".git"))).isSymbolicLink()) fail("unsafe_git_link"); @@ -579,7 +587,7 @@ async function workspaceArchive(input: { } } -export function captureRemoteWorkspaceDirectory(input: { cwd: string; archivePath: string; manifestPath?: string; exclude?: string[]; signal?: AbortSignal }) { +export function captureRemoteWorkspaceDirectory(input: { cwd: string; archivePath: string; manifestPath?: string; exclude?: string[]; requireNoGit?: boolean; signal?: AbortSignal }) { return workspaceArchive({ ...input, kind: "capture" }); } @@ -617,6 +625,7 @@ export function remoteWorkspaceCaptureFailureMessage(error: unknown): string | n const diskFull = "remote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the workspace"; const validation = [ "remote_workspace_archive_changed", "remote_workspace_archive_unsupported_filename", + "remote_workspace_archive_online_source_git_requires_shared_fence", "remote_workspace_archive_unsafe_git_link", "remote_workspace_archive_invalid_git_repository", "remote_workspace_archive_unsafe_symlink", "remote_workspace_archive_unsupported_entry", "remote_workspace_archive_source_missing", diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index c617107f48b..ef07a03aa08 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -8,6 +8,7 @@ import { getStorageService } from "../storage/index.js"; import type { StorageService } from "../storage/types.js"; import { sanitizeRuntimeServiceBaseEnv } from "./runtime-service-env.js"; import { verifiedRecordedIsolatedRemoteSource } from "./isolated-remote-workspace-source.js"; +import { assertCurrentWorkspaceSourceHeld, isolatedTaskSourceIdentity, managedProjectSourceIdentity, withWorkspaceSourceLocks } from "./workspace-source-lock.js"; export { sanitizeRuntimeServiceBaseEnv } from "./runtime-service-env.js"; import { execFile, spawn, type ChildProcess } from "node:child_process"; import { existsSync, lstatSync, readdirSync, readFileSync, realpathSync } from "node:fs"; @@ -5165,6 +5166,7 @@ function renderRuntimeServiceEnv(input: { function resolveRuntimeServiceReuseIdentity(input: { service: Record; workspace: RealizedExecutionWorkspace; + executionWorkspaceId?: string | null; agent: ExecutionWorkspaceAgentRef; issue: ExecutionWorkspaceIssueRef | null; adapterEnv: Record; @@ -5209,6 +5211,8 @@ function resolveRuntimeServiceReuseIdentity(input: { stableStringify({ scopeType: input.scopeType, scopeId: input.scopeId, + ...(input.scopeType === "run" && input.executionWorkspaceId + ? { executionWorkspaceId: input.executionWorkspaceId } : {}), serviceName, command, cwd: serviceCwd, @@ -6204,6 +6208,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P const identity = resolveRuntimeServiceReuseIdentity({ service: input.service, workspace: input.workspace, + executionWorkspaceId: input.executionWorkspaceId, agent: input.agent, issue: input.issue, adapterEnv: input.adapterEnv, @@ -6675,6 +6680,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P } try { + if (input.db) await assertCurrentWorkspaceSourceHeld(serviceCwd); await ensureServerWorkspaceLinksCurrent(serviceCwd, { onLog: input.onLog, }); @@ -6997,6 +7003,7 @@ async function prepareRuntimeProvisioning( const identity = resolveRuntimeServiceReuseIdentity({ service: input.service, workspace: input.workspace, + executionWorkspaceId: input.executionWorkspaceId, agent: input.agent, issue: input.issue, adapterEnv: input.adapterEnv, @@ -7641,6 +7648,7 @@ async function ensureRuntimeServicesForRunInvocation( const reuseKey = resolveRuntimeServiceReuseIdentity({ service, workspace: input.workspace, + executionWorkspaceId: input.executionWorkspaceId, agent: input.agent, issue: input.issue, adapterEnv: input.adapterEnv, @@ -7741,6 +7749,7 @@ function resolveRuntimeStartMutexPlan(input: { const reuseKey = resolveRuntimeServiceReuseIdentity({ service, workspace: input.workspace, + executionWorkspaceId: input.executionWorkspaceId, agent: input.agent, issue: input.issue, adapterEnv: input.adapterEnv, @@ -7789,6 +7798,23 @@ async function withRuntimeStartMutexes( return await acquire(0); } +async function withRuntimeServiceSourceLocks(input: { + db?: Db; workspace: RealizedExecutionWorkspace; agent: ExecutionWorkspaceAgentRef; + issue: ExecutionWorkspaceIssueRef | null; adapterEnv: Record; services: Record[]; +}, action: () => Promise): Promise { + if (!input.db || input.services.length === 0) return action(); + const paths = [input.workspace.cwd, input.workspace.baseCwd, ...input.services.map(service => + resolveRuntimeServiceReuseIdentity({ ...input, service, scopeType: "project_workspace", scopeId: input.workspace.workspaceId }).serviceCwd)]; + const projectIds: string[] = [], issueIds: string[] = []; + for (const cwd of new Set(paths)) { + const projectId = await managedProjectSourceIdentity(input.agent.companyId, cwd); + const issueId = await isolatedTaskSourceIdentity(input.agent.companyId, cwd); + if (projectId) projectIds.push(projectId); + if (issueId) issueIds.push(issueId); + } + return withWorkspaceSourceLocks(input.db, { companyId: input.agent.companyId, projectIds, issueIds }, action); +} + export async function ensureRuntimeServicesForRun( input: EnsureRuntimeServicesForRunInput, ): Promise { @@ -7810,7 +7836,7 @@ export async function ensureRuntimeServicesForRun( try { return await withRuntimeStartMutexes( mutexPlan.ownerKeys, - () => ensureRuntimeServicesForRunInvocation(input), + () => withRuntimeServiceSourceLocks({ ...input, services }, () => ensureRuntimeServicesForRunInvocation(input)), ); } finally { releaseRuntimeReplacementClaims(mutexPlan.replacementReuseKeys); @@ -7875,6 +7901,7 @@ async function startRuntimeServicesForWorkspaceControlUnlocked( const reuseKey = resolveRuntimeServiceReuseIdentity({ service, workspace: input.workspace, + executionWorkspaceId: input.executionWorkspaceId, agent: input.actor, issue: input.issue, adapterEnv: input.adapterEnv, @@ -8074,6 +8101,7 @@ async function startRuntimeServicesForWorkspaceControlInvocation( const reuseKey = resolveRuntimeServiceReuseIdentity({ service, workspace: input.workspace, + executionWorkspaceId: input.executionWorkspaceId, agent: input.actor, issue: input.issue, adapterEnv: input.adapterEnv, @@ -8259,10 +8287,8 @@ export async function startRuntimeServicesForWorkspaceControl( try { return await withRuntimeStartMutexes( mutexPlan.ownerKeys, - async () => { - if (!input.db || !input.executionWorkspaceId) { - return startRuntimeServicesForWorkspaceControlInvocation(input); - } + () => withRuntimeServiceSourceLocks({ ...input, agent: input.actor, services }, async () => { + if (!input.db || !input.executionWorkspaceId) return startRuntimeServicesForWorkspaceControlInvocation(input); // Startup and direct callers can bypass the HTTP operation recorder. // Keep migration fenced through provision/bind/persistence, including the // HTTPS path which intentionally commits its reservation before binding. @@ -8284,7 +8310,7 @@ export async function startRuntimeServicesForWorkspaceControl( } return startRuntimeServicesForWorkspaceControlInvocation(input); }); - }, + }), ); } finally { releaseRuntimeReplacementClaims(mutexPlan.replacementReuseKeys); diff --git a/server/src/services/workspace-source-lock.ts b/server/src/services/workspace-source-lock.ts new file mode 100644 index 00000000000..b9f0ac2b7f3 --- /dev/null +++ b/server/src/services/workspace-source-lock.ts @@ -0,0 +1,107 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { withDedicatedDbConnection, type Db } from "@paperclipai/db"; +import { sql } from "drizzle-orm"; +import path from "node:path"; +import { resolvePaperclipInstanceRoot } from "../home-paths.js"; +import { canonicalRepositoryRoot } from "./execution-workspace-source-roots.js"; +import { withWorkspaceLockKeepalive } from "./native-runtime/native-workspace-lock-keepalive.js"; + +type SourceTransaction = Parameters[0]>[0]; +type SourceLockScope = { db: Db; companyId: string; keys: Set; transaction: SourceTransaction }; +const sourceLocks = new AsyncLocalStorage(); +export class IsolatedRepositorySourceBusy extends Error {} + +export function withIsolatedRepositorySourceLock(db: Db, companyId: string, issueId: string, + action: () => Promise, tryOnly = false): Promise { + return withWorkspaceSourceLocks(db, { companyId, issueIds: [issueId] }, action, tryOnly); +} + +export async function withWorkspaceSourceLocks(db: Db, + input: { companyId: string; issueIds?: readonly string[]; projectIds?: readonly string[] }, + action: (transaction: SourceTransaction) => Promise, tryOnly = false): Promise { + const keys = [...new Set([ + ...(input.issueIds ?? []).map(id => `isolated_repository_source:${input.companyId}:${id}`), + ...(input.projectIds ?? []).map(id => `managed_project_source:${input.companyId}:${id}`), + ])].sort(); + if (keys.length > 128) throw new Error("workspace_source_scope_limit"); + const held = sourceLocks.getStore(); + const acquire = async (scope: SourceLockScope, nonblocking: boolean) => { + for (const key of keys) { + if (scope.keys.has(key)) continue; + const lock = nonblocking ? sql`pg_try_advisory_xact_lock(hashtextextended(${key}, 0))` + : sql`pg_advisory_xact_lock(hashtextextended(${key}, 0))`; + const rows = await scope.transaction.execute(sql`select ${lock} as acquired`); + if (nonblocking && !rows[0]?.acquired) throw new IsolatedRepositorySourceBusy("workspace_source_busy"); + scope.keys.add(key); + } + }; + if (held?.db === db) { + if (new Set([...held.keys, ...keys]).size > 128) throw new Error("workspace_source_scope_limit"); + if (held.companyId !== input.companyId) throw new Error("workspace_source_company_mismatch"); + const onlyProjectKeys = keys.every(key => key.startsWith("managed_project_source:")); + const alreadyHasProject = [...held.keys].some(key => key.startsWith("managed_project_source:")); + await acquire(held, !onlyProjectKeys || alreadyHasProject); + return action(held.transaction); + } + return withDedicatedDbConnection(db, connection => connection.transaction(async transaction => { + const scope = { db, companyId: input.companyId, transaction, keys: new Set() }; + await acquire(scope, tryOnly); + const rows = await transaction.execute(sql`select setting::bigint as idle_timeout_ms from pg_settings where name = 'idle_in_transaction_session_timeout'`); + return withWorkspaceLockKeepalive(transaction, Number(rows[0]?.idle_timeout_ms), + () => sourceLocks.run(scope, () => action(transaction))); + })); +} + +export async function managedProjectSourceIdentity(companyId: string, cwd: string): Promise { + const root = await canonicalRepositoryRoot(path.join(resolvePaperclipInstanceRoot(), "projects")); + const target = await canonicalRepositoryRoot(cwd); + const relative = path.relative(root, target); + if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) return null; + const [owner, projectId] = relative.split(path.sep); + if (owner !== companyId) throw new Error("workspace_source_company_mismatch"); + return projectId && /^[a-f0-9-]{36}$/.test(projectId) ? projectId : null; +} + +export async function isolatedTaskSourceIdentity(companyId: string, cwd: string): Promise { + const root = await canonicalRepositoryRoot(resolvePaperclipInstanceRoot()); + const relative = path.relative(root, await canonicalRepositoryRoot(cwd)).split(path.sep); + if (!["isolated-workspaces", "isolated-workspace-attempts"].includes(relative[0])) return null; + if (relative[1] !== companyId) throw new Error("workspace_source_company_mismatch"); + return relative[2] && /^[a-f0-9-]{36}$/.test(relative[2]) ? relative[2] : null; +} + +export async function retainManagedProjectSource(db: Db, companyId: string, cwd: string): Promise { + const projectId = await managedProjectSourceIdentity(companyId, cwd); + if (!projectId) return; + if (sourceLocks.getStore()?.db !== db) throw new Error("workspace_source_admission_scope_required"); + await withWorkspaceSourceLocks(db, { companyId, projectIds: [projectId] }, async () => undefined, true); +} + +export async function retainCurrentWorkspaceSource(cwd: string): Promise { + const scope = sourceLocks.getStore(); + if (!scope) return; + const instance = await canonicalRepositoryRoot(resolvePaperclipInstanceRoot()); + const task = path.relative(instance, await canonicalRepositoryRoot(cwd)).split(path.sep); + if (["isolated-workspaces", "isolated-workspace-attempts"].includes(task[0]) && task.length >= 3) { + const issueId = await isolatedTaskSourceIdentity(scope.companyId, cwd); + if (issueId) await withWorkspaceSourceLocks(scope.db, { companyId: scope.companyId, issueIds: [issueId] }, async () => undefined, true); + return; + } + const root = await canonicalRepositoryRoot(path.join(resolvePaperclipInstanceRoot(), "projects")); + const target = await canonicalRepositoryRoot(cwd), relative = path.relative(root, target); + if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) return; + const [companyId, projectId] = relative.split(path.sep); + if (!companyId || !projectId || !/^[a-f0-9-]{36}$/.test(companyId) || !/^[a-f0-9-]{36}$/.test(projectId)) return; + await retainManagedProjectSource(scope.db, companyId, cwd); +} + +export async function assertCurrentWorkspaceSourceHeld(cwd: string): Promise { + const scope = sourceLocks.getStore(); + if (!scope) throw new Error("workspace_source_admission_scope_required"); + const projectId = await managedProjectSourceIdentity(scope.companyId, cwd); + const issueId = await isolatedTaskSourceIdentity(scope.companyId, cwd); + if (projectId && !scope.keys.has(`managed_project_source:${scope.companyId}:${projectId}`) + || issueId && !scope.keys.has(`isolated_repository_source:${scope.companyId}:${issueId}`)) { + throw new IsolatedRepositorySourceBusy("workspace_source_busy"); + } +}