diff --git a/doc/plans/2026-11-07-ownership-inventory.md b/doc/plans/2026-11-07-ownership-inventory.md index 1c24479d7f4..9d0d4d82888 100644 --- a/doc/plans/2026-11-07-ownership-inventory.md +++ b/doc/plans/2026-11-07-ownership-inventory.md @@ -351,7 +351,7 @@ Scope: workspace operations (#19807 intents), runtime leases, runtime services, | Execution workspace runtime lease | `execution_workspace_runtime_leases` (row existence) | workspace-runtime-leases.ts:194-209 (create), :235-247 (renew), :255-273 (reclaim). Called only from routes/execution-workspaces.ts:567 | row present | `owner_key` `issue:`/`run:`/`agent:` (+ FK columns). No process identity | `release` :291-313. In production it is called only with `force` on archive (routes/execution-workspaces.ts:1270). Otherwise the lease is reclaimed lazily at the next claim (:250-283) | `evaluateStaleOwner` :85-119: issue hidden or terminal, run terminal, or **TTL/age** 30 min (:39, :90). An agent-scoped owner has TTL only (:116-118) | ownership | 1, 2 | A, C | | Workspace runtime service (local process) | `workspace_runtime_services.status` | workspace-runtime.ts (insert line unverified) | `provisioning`, `starting`, `running` | `startedByRunId`, `ownerAgentId`. The pid lives in a local service registry file, not on the row | Startup `reconcilePersistedRuntimeServicesOnStartup` workspace-runtime.ts:8636, which adopts or stops each service (index.ts:1101, not awaited) | Registry record match (workspace-runtime.ts:8781-8797) plus `isPidAlive` local-service-supervisor.ts:218, :245-250. Same host only | ownership | 1, 2 | A, C | | Remote workspace writer lane | `execution_workspaces.metadata.remoteWorkspace.writerRunId` (+ `epoch`) | remote-workspaces.ts:230 (epoch+1), :243 (new) | non-null writerRunId | writer run id + epoch | Final save sets it to null: remote-workspaces.ts:780-781. Migration: remote-workspace-migration.ts:516 | `readRemoteWorkspaceWriterHold` remote-workspace-writer-hold.ts:25-79: run status (:42), lease termination/stop receipts (:53-56), recovery-save progress (:63-76). The waiter gives up after **TTL/age** 60 min of writer inactivity (heartbeat/workspace-preparation.ts:1050-1051, run-dispatch/domain/wake-context.ts:36) | ownership | 1, 2, 3, 7 | A, E | -| Remote workspace unsaved-files flag | `execution_workspaces.metadata.remoteWorkspace.dirty` | remote-workspaces.ts:780-781 (`dirty: boundary !== "final"`) | `dirty=true` | writerRunId (same object) | Final save :781. Failed-run recovery native-runtime/native-failed-workspace-recovery.ts:52-60 (pins/saves) | Writer-hold rule (above). Lease termination receipt | workspace-preservation | 1, 2 | C | +| Remote workspace unsaved-files flag | `execution_workspaces.metadata.remoteWorkspace.dirty` | remote-workspaces.ts:780-781 (`dirty: boundary !== "final"`) | `dirty=true` | writerRunId (same object) | Final save :781. Failed-run recovery native-runtime/native-failed-workspace-recovery.ts:52-60 (pins/saves). A writer that died during preparation (terminal, no resolved runtime, no native finalization, no save in flight) is ended at the last committed save by the ownership reconciler (`settlePreparationOnlyRemoteWorkspaceEpoch`) | Writer-hold rule (above). Lease termination receipt | workspace-preservation | 1, 2 | C | | Remote snapshot ledger | `execution_workspace_snapshots.status` | remote-workspace-snapshot-storage.ts:73-83 (`pending`, deleteAfter = now + 7 d) | `pending`, `uploaded` | `source_operation_id`, `source_run_id` | Commit remote-workspaces.ts:782-784. Orphans become `delete_requested` by **TTL/age** 7 d from createdAt (remote-workspace-snapshot-retention.ts:9, :113-122), deferred while ops are running (:36) | none (age) | workspace-preservation | 2 | C | | Git restore point | `execution_workspace_restore_points.status` | git-restore-point-storage.ts:146-155 | `pending` (and `expiring`) | none (workspace + generation) | → `active` :187-188. → `superseded` :141-145, :182-185, :208-216 | none found | workspace-preservation | 1, 2 | C | | Git cache expiration intent | `execution_workspaces.metadata.gitCacheExpiration.completed` | git-workspace-expiration.ts:384-390, :401-406 | `completed=false` | `lockToken` (random, :384, :405). File locks git-workspace-expiration-locks.ts:72-76 | Deleting the key on completion git-workspace-expiration.ts:454. Restore/rehome deletes it (execution-workspaces.ts:4516, isolated-repository-rehome.ts:485) | Transaction advisory lock `execution_workspace_lifecycle:` (:302-303). Resumable by any worker using the same token (recover: git-workspace-expiration-locks.ts:57-63) | workspace-preservation | 1 | B, C | diff --git a/server/src/__tests__/ownership-reconciliation-report.test.ts b/server/src/__tests__/ownership-reconciliation-report.test.ts index 1067046bb5b..e9e92261278 100644 --- a/server/src/__tests__/ownership-reconciliation-report.test.ts +++ b/server/src/__tests__/ownership-reconciliation-report.test.ts @@ -12,6 +12,8 @@ import { companies, createDb, environmentLeases, + environments, + executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issueComments, @@ -34,6 +36,9 @@ import { } from "../services/recovery/ownership-invariant-report.js"; import { OWNABLE_KINDS } from "../services/recovery/ownership-registry.js"; import { trackRemoteWorkspaceSaveInvocationForTests } from "../services/remote-workspaces.js"; +import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; +import { preparedRemoteWorkspaceSource } from "../services/remote-workspace-lifecycle.js"; +import { readRemoteWorkspaceWriterHold } from "../services/remote-workspace-writer-hold.js"; import { workspaceOperationService } from "../services/workspace-operations.js"; const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HISTORY_TEST_DATABASE_URL"); @@ -146,7 +151,7 @@ describe("report-only ownership reconciliation", () => { expect(new Set(names).size).toBe(names.length); expect(names.length).toBeGreaterThanOrEqual(40); expect(OWNABLE_KINDS.filter(kind => kind.settle !== null).map(kind => kind.id)) - .toEqual(["native_accepted_result_lease", "native_finalizer_lease", "native_cleanup_lease", "sandbox_creation_intent", "environment_run_lease_active", "environment_teardown_claim", "environment_retained_orphan_sandbox", "environment_login_probe_lease", "workspace_runtime_control_operation", "native_workspace_finalize_attempt", "workspace_command_operation", "remote_snapshot_save", "execution_workspace_runtime_lease", "legacy_shutdown_settlement_fence", "copyback_owner_hold", "observed_workspace_preservation_fence", "wake_request_claimed", "legacy_controller_lease", "legacy_interrupted_preparation_hold", "issue_execution_lock", "issue_checkout_lock", "agent_lifecycle_lease"]); + .toEqual(["native_accepted_result_lease", "native_finalizer_lease", "native_cleanup_lease", "sandbox_creation_intent", "environment_run_lease_active", "environment_teardown_claim", "environment_retained_orphan_sandbox", "environment_login_probe_lease", "workspace_runtime_control_operation", "native_workspace_finalize_attempt", "workspace_command_operation", "remote_snapshot_save", "execution_workspace_runtime_lease", "remote_workspace_unsaved_files", "legacy_shutdown_settlement_fence", "copyback_owner_hold", "observed_workspace_preservation_fence", "wake_request_claimed", "legacy_controller_lease", "legacy_interrupted_preparation_hold", "issue_execution_lock", "issue_checkout_lock", "agent_lifecycle_lease"]); expect(OWNABLE_KINDS.filter(kind => !kind.listInProgress).every(kind => kind.coverageGap)).toBe(true); }); @@ -452,6 +457,76 @@ describe("report-only ownership reconciliation", () => { expect(liveRun.status).toBe("running"); }); + it("ends a dirty remote epoch whose writer died during preparation, and keeps every other writer's files", async () => { + const runs = { preparing: randomUUID(), resolved: randomUUID(), executing: randomUUID(), live: randomUUID(), saving: randomUUID(), + invoked: randomUUID(), spawned: randomUUID() }; + const run = (id: string, status: string, extra: Record = {}) => ({ id, companyId, agentId, status, + invocationSource: "manual", executionStage: "preparing", ...(status === "running" ? {} : { finishedAt: minutesAgo(2), errorCode: "process_lost" }), ...extra }); + await db.insert(heartbeatRuns).values([ + run(runs.preparing, "failed"), + run(runs.resolved, "failed", { runtimeMode: "native", runtimeModeResolvedAt: minutesAgo(3) }), + run(runs.executing, "failed", { executionStage: "executing" }), + run(runs.live, "running"), + run(runs.saving, "failed"), + run(runs.invoked, "failed"), + run(runs.spawned, "failed", { processPid: 4_194_303 }), + ]); + await db.insert(heartbeatRunEvents).values({ companyId, runId: runs.invoked, agentId, seq: 1, eventType: "adapter.invoke" }); + const workspaceFor = Object.fromEntries(Object.keys(runs).map(name => [name, randomUUID()])) as Record; + await db.insert(executionWorkspaces).values(Object.entries(runs).map(([name, writerRunId]) => ({ + id: workspaceFor[name as keyof typeof runs], companyId, mode: "isolated_workspace", strategyType: "test", name, + metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null, + source: { kind: "git", repoUrl: "https://example.invalid/repo.git", ref: null }, dirty: true, writerRunId, + location: { environmentId: randomUUID(), leaseId: randomUUID(), providerLeaseId: "sandbox-1", remoteCwd: "/workspace/root" } } }, + }))); + await db.insert(workspaceOperations).values({ companyId, executionWorkspaceId: workspaceFor.saving, heartbeatRunId: runs.saving, + phase: "remote_workspace_snapshot", status: "running", metadata: { epoch: 1, boundary: "initial" } }); + const authorities = async () => Object.fromEntries((await db.select().from(executionWorkspaces)) + .filter(row => Object.values(workspaceFor).includes(row.id)) + .map(row => [row.name, readRemoteWorkspaceAuthority(row.metadata)])); + + const first = await runOwnershipReconciliationPass(db, { state: createOwnershipReportState() }); + expect(kindReport(first, "remote_workspace_unsaved_files")).toMatchObject({ settled: 1 }); + const after = await authorities(); + expect(after.preparing).toMatchObject({ epoch: 2, dirty: false, writerRunId: null, location: null, snapshotId: null }); + for (const kept of ["resolved", "executing", "live", "saving", "invoked", "spawned"] as const) + expect(after[kept], kept).toMatchObject({ epoch: 1, dirty: true, writerRunId: runs[kept] }); + const unsavedFiles = OWNABLE_KINDS.find(kind => kind.id === "remote_workspace_unsaved_files")!; + expect((await recheckSettlingKinds(db, { kinds: [unsavedFiles] })).watched).toBe(2); + const [abandoned] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceFor.preparing)); + expect(abandoned.metadata?.remoteWorkspaceAbandonedEpochs).toEqual([expect.objectContaining({ epoch: 1, runId: runs.preparing })]); + }); + + it("releases the lease and frees the workspace for the retry once a preparation-only epoch ends", async () => { + const runId = randomUUID(), issueId = randomUUID(), workspaceId = randomUUID(), leaseId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Remote task", status: "in_progress" }); + const [environment] = await db.insert(environments).values({ name: `Sandbox ${randomUUID()}`, driver: "sandbox" }).returning(); + const location = { environmentId: environment.id, leaseId, providerLeaseId: `sandbox-${randomUUID()}`, remoteCwd: "/workspace/task" }; + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "failed", invocationSource: "manual", + executionStage: "preparing", finishedAt: minutesAgo(2), errorCode: "process_lost", contextSnapshot: { issueId }, + runnerProfileJson: { remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", workspaceId, leaseId, + providerLeaseId: location.providerLeaseId, remoteCwd: location.remoteCwd, epoch: 1, state: "prepared", snapshotId: null } } }); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issueId, mode: "isolated_workspace", + strategyType: "test", name: "Remote task workspace", + metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null, + source: { kind: "git", repoUrl: "https://example.invalid/repo.git", ref: null }, dirty: true, writerRunId: runId, location } } }); + const [lease] = await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId: environment.id, issueId, + heartbeatRunId: runId, executionWorkspaceId: workspaceId, status: "active", leasePolicy: "ephemeral", provider: "daytona", + providerLeaseId: location.providerLeaseId, metadata: { remoteCwd: location.remoteCwd } }).returning(); + const authority = async () => readRemoteWorkspaceAuthority((await db.select().from(executionWorkspaces) + .where(eq(executionWorkspaces.id, workspaceId)))[0]!.metadata)!; + expect(await preparedRemoteWorkspaceSource(db, lease)).not.toBeNull(); + expect(await readRemoteWorkspaceWriterHold(db, { companyId, authority: await authority(), runId: randomUUID() })) + .toMatchObject({ kind: "held" }); + + await runOwnershipReconciliationPass(db, { state: createOwnershipReportState() }); + await runOwnershipReconciliationPass(db, { state: createOwnershipReportState() }); + expect(await preparedRemoteWorkspaceSource(db, lease)).toBeNull(); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)))[0]!.status).not.toBe("active"); + expect(await readRemoteWorkspaceWriterHold(db, { companyId, authority: await authority(), runId: randomUUID() })) + .toEqual({ kind: "free" }); + }); + it("settles a workspace command whose process died mid-command, and keeps live or unprovable ones", async () => { const run = (id: string, status: string) => ({ id, companyId, agentId, status, invocationSource: "manual", ...(status === "running" ? {} : { finishedAt: minutesAgo(2) }) }); diff --git a/server/src/services/recovery/ownership-registry.ts b/server/src/services/recovery/ownership-registry.ts index a4865834760..d576c4f48ea 100644 --- a/server/src/services/recovery/ownership-registry.ts +++ b/server/src/services/recovery/ownership-registry.ts @@ -38,6 +38,7 @@ import { } from "../native-runtime/native-accepted-result-lease-settlement.js"; import { settleClaimedWakeRequest } from "./wake-request-claimed-settlement.js"; import { settleAbandonedRemoteSnapshotSave, settleAbandonedWorkspaceCommandOperation, settleAbandonedWorkspaceFinalizeAttempt } from "./workspace-finalize-attempt-settlement.js"; +import { settlePreparationOnlyRemoteWorkspaceEpoch } from "./remote-workspace-unsaved-files-settlement.js"; import { LOCAL_WORKSPACE_COMMAND_PHASES, runtimeControlOperationOwnerStamp, settleRuntimeControlOperation } from "../workspace-operations.js"; import { copybackOwnerHoldOwner, settleCopybackOwnerHold } from "./copyback-owner-hold.js"; import { applyObservedFenceProviderProof, observedWorkspaceFenceOwner, settleObservedWorkspaceFence } from "./observed-workspace-fence.js"; @@ -930,7 +931,7 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ }, { id: "remote_workspace_unsaved_files", title: "Remote workspace unsaved-files flag", obligation: "workspace_preservation", - storage: "database", settle: null, invariantChecks: [1, 2], + storage: "database", settle: settlePreparationOnlyRemoteWorkspaceEpoch, invariantChecks: [1, 2], listInProgress: async (db, limit, offset) => { const rows = await db.select({ id: executionWorkspaces.id, companyId: executionWorkspaces.companyId, metadata: executionWorkspaces.metadata, updatedAt: executionWorkspaces.updatedAt }).from(executionWorkspaces) diff --git a/server/src/services/recovery/remote-workspace-unsaved-files-settlement.ts b/server/src/services/recovery/remote-workspace-unsaved-files-settlement.ts new file mode 100644 index 00000000000..1dbdf5c02cf --- /dev/null +++ b/server/src/services/recovery/remote-workspace-unsaved-files-settlement.ts @@ -0,0 +1,60 @@ +import { and, eq, inArray, like, or, sql } from "drizzle-orm"; +import { executionWorkspaces, heartbeatRunEvents, heartbeatRuns, nativeRunFinalizations, workspaceOperations, type Db } from "@paperclipai/db"; +import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; +import type { OwnerProof } from "../owner-proof.js"; +import { abandonUnverifiedRemoteWorkspaceEpoch } from "../remote-workspace-epoch-abandonment.js"; +import { notifyTerminalWorkspaceWork } from "../terminal-workspace-work.js"; +import type { InProgressRecord, SettleOutcome } from "./ownership-registry.js"; + +const TERMINAL_RUN_STATUSES = ["succeeded", "failed", "cancelled", "timed_out", "interrupted"]; +const WORK_STARTED_EVENT_TYPES = ["adapter.invoke", "adapter.progress"]; +const WORK_STARTED_EVENT_PREFIXES = ["turn.", "provider.", "native.process_"]; + +export async function settlePreparationOnlyRemoteWorkspaceEpoch(db: Db, record: InProgressRecord, + context: { now: Date; proof: OwnerProof }): Promise { + if (record.owner.kind !== "run" || !record.runId || !record.companyId) return { settled: false, reason: "owner_not_provable" }; + const runId = record.runId, companyId = record.companyId; + return db.transaction(async (transaction): Promise => { + const tx = transaction as unknown as Db; + const [run] = await tx.select({ status: heartbeatRuns.status, executionStage: heartbeatRuns.executionStage, + runtimeModeResolvedAt: heartbeatRuns.runtimeModeResolvedAt, processPid: heartbeatRuns.processPid, + processGroupId: heartbeatRuns.processGroupId, nativeSessionId: heartbeatRuns.nativeSessionId, + runnerInstanceId: heartbeatRuns.runnerInstanceId }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))).for("update"); + if (!run || !TERMINAL_RUN_STATUSES.includes(run.status)) return { settled: false, reason: "owner_run_active" }; + if (run.runtimeModeResolvedAt || run.executionStage !== "preparing") return { settled: false, reason: "writer_reached_runtime", watch: false }; + const [finalization] = await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations) + .where(and(eq(nativeRunFinalizations.companyId, companyId), eq(nativeRunFinalizations.runId, runId))).limit(1); + if (finalization || run.processPid || run.processGroupId || run.nativeSessionId || run.runnerInstanceId) { + return { settled: false, reason: "writer_reached_runtime", watch: false }; + } + const [workStarted] = await tx.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( + eq(heartbeatRunEvents.runId, runId), or(inArray(heartbeatRunEvents.eventType, WORK_STARTED_EVENT_TYPES), + ...WORK_STARTED_EVENT_PREFIXES.map(prefix => like(heartbeatRunEvents.eventType, `${prefix}%`))))).limit(1); + if (workStarted) return { settled: false, reason: "writer_reached_runtime", watch: false }; + const [workspace] = await tx.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces) + .where(and(eq(executionWorkspaces.companyId, companyId), eq(executionWorkspaces.id, record.recordId))); + const authority = readRemoteWorkspaceAuthority(workspace?.metadata); + if (!authority?.dirty || authority.writerRunId !== runId || !authority.location) { + return { settled: false, reason: "unsaved_files_no_longer_held" }; + } + const [save] = await tx.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, companyId), eq(workspaceOperations.executionWorkspaceId, record.recordId), + eq(workspaceOperations.phase, "remote_workspace_snapshot"), inArray(workspaceOperations.status, ["running", "pending"]), + sql`${workspaceOperations.metadata}->>'epoch' = ${String(authority.epoch)}`)).limit(1); + if (save) return { settled: false, reason: "save_in_flight" }; + await notifyTerminalWorkspaceWork(tx, `workspace:${record.recordId}`); + try { + await abandonUnverifiedRemoteWorkspaceEpoch(transaction, { + companyId, workspaceId: record.recordId, runId, epoch: authority.epoch, snapshotId: authority.snapshotId, + location: authority.location, + }); + } catch (error) { + if (error instanceof Error && error.message === "remote_workspace_authority_changed") { + return { settled: false, reason: "unsaved_files_no_longer_held" }; + } + throw error; + } + return { settled: true, reason: context.proof.reason }; + }); +} diff --git a/server/src/services/remote-workspace-save-concurrency.test.ts b/server/src/services/remote-workspace-save-concurrency.test.ts index 29eae98db7d..9b54771b720 100644 --- a/server/src/services/remote-workspace-save-concurrency.test.ts +++ b/server/src/services/remote-workspace-save-concurrency.test.ts @@ -186,6 +186,66 @@ describe.skipIf(!support.supported)("remote workspace save parent lock order", ( } }, 30_000); + it("lets a retry prepare the workspace again after its first writer was killed during preparation", async () => { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(), retryId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Killed preparation", issuePrefix: `K${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "paperclip_runner" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Prepare twice", status: "in_progress", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "running", executionStage: "preparing", + contextSnapshot: { issueId } }); + const [environment] = await db.insert(environments).values({ name: runId, driver: "sandbox" }).returning(); + const storage = { ...createStorageService(createLocalDiskStorageProvider(path.join(root, runId, "objects"))), provider: "s3" as const }; + const allocate = async (heartbeatRunId: string, directory: string, killSave = false) => { + const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, heartbeatRunId, environmentId: environment.id, + provider: "daytona", providerLeaseId: randomUUID(), status: "active" }).returning(); + const remoteCwd = path.join(root, runId, directory); + await mkdir(remoteCwd, { recursive: true }); + const transfer: NonNullable["syncIn"]> = async operations => { + for (const operation of operations) for (const file of operation.files) { + await mkdir(path.dirname(file.targetPath), { recursive: true }); + await cp(file.sourcePath, file.targetPath, { recursive: file.kind === "directory" }); + } + return { operations: operations.map(operation => ({ operationId: operation.operationId, + filesTransferred: operation.files.length, bytesTransferred: 0 })) }; + }; + const target: AdapterSandboxExecutionTarget = { kind: "remote", transport: "sandbox", leaseId: lease.id, remoteCwd, timeoutMs: 30_000, + runner: { syncIn: transfer, syncOut: killSave ? async () => { throw new Error("controller killed"); } : transfer, + execute: async input => { + const result = await exec(input.command, input.args ?? [], { cwd: input.cwd ?? remoteCwd, + env: { ...process.env, ...input.env }, timeout: input.timeoutMs, maxBuffer: 8 * 1024 * 1024 }); + return { exitCode: 0, stdout: result.stdout, stderr: result.stderr, signal: null, timedOut: false, + pid: process.pid, startedAt: new Date().toISOString() }; + } } }; + return { lease: lease as EnvironmentLease, target, remoteCwd }; + }; + const first = await allocate(runId, "first", true); + const reserved = await reserveRemoteWorkspace({ db, storage, companyId, issueId, runId, agentId, environmentId: environment.id, + projectId: null, projectWorkspaceId: null, existingWorkspaceId: null, source: { kind: "empty" } }); + await expect(prepareRemoteWorkspace({ db, storage, workspace: reserved, runId, lease: first.lease, target: first.target })) + .rejects.toThrow(); + const workspaceAuthority = async () => readRemoteWorkspaceAuthority((await db.select().from(executionWorkspaces) + .where(eq(executionWorkspaces.id, reserved.id)))[0]!.metadata)!; + expect(await workspaceAuthority()).toMatchObject({ epoch: 1, dirty: true, writerRunId: runId, snapshotId: null }); + + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "process_lost", finishedAt: new Date() }) + .where(eq(heartbeatRuns.id, runId)); + await db.insert(heartbeatRuns).values({ id: retryId, companyId, agentId, issueId, status: "running", executionStage: "preparing", + contextSnapshot: { issueId, retryOfRunId: runId } }); + await runOwnershipReconciliationPass(db, { state: createOwnershipReportState() }); + await runOwnershipReconciliationPass(db, { state: createOwnershipReportState() }); + expect(await workspaceAuthority()).toMatchObject({ epoch: 2, dirty: false, writerRunId: null, location: null, snapshotId: null }); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, first.lease.id)))[0]!.status).not.toBe("active"); + + const second = await allocate(retryId, "second"); + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, reserved.id)); + const retried = await reserveRemoteWorkspace({ db, storage, companyId, issueId, runId: retryId, agentId, + environmentId: environment.id, projectId: null, projectWorkspaceId: null, existingWorkspaceId: workspace.id, source: { kind: "empty" } }); + const reprepared = await prepareRemoteWorkspace({ db, storage, workspace: retried, runId: retryId, + lease: second.lease, target: second.target }); + expect(await workspaceAuthority()).toMatchObject({ epoch: 3, dirty: true, writerRunId: retryId, snapshotId: expect.any(String) }); + expect(path.relative(first.remoteCwd, reprepared.remoteRoot).startsWith("..")).toBe(true); + }, 30_000); + it("releases database locks while a provisioning command is in flight", async () => { const f = await preparedFailure(); await db.update(heartbeatRuns).set({ status: "running", finishedAt: null }).where(eq(heartbeatRuns.id, f.runId));