From 7c56147bb387bf8afc763177bd6716c35df6b35e Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:50:33 -0600 Subject: [PATCH] fix(workspaces): a busy-workspace holder whose controller lease is held is live, however silent (#20023) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Two runs must never write one workspace at once. A run that finds a workspace held (a remote workspace's writer run, or a running holder of a shared local tree) waits behind it > - The holder counted as live only while it produced output: after 60 minutes of silence it was "stale" > - For the shared local tree, that meant the waiter went ahead while the silent holder might still be working, which is a second writer. > - For a remote workspace, the waiter failed instead of waiting. > - Silence is not proof a run stopped. A long tool call can be silent and alive > - The durable ownership plan proves ownership by identity or by a held fence, never by age > - This pull request treats a holder whose controller still renews its lease (the legacy controller lease or the native coordinator lease) as live, however silent. The 60-minute rule applies only once that fence lapses > - The benefit is that a silent but live holder is never overtaken. A dead holder is still settled by the run reapers, which wake the waiters ## Linked Issues or Issue Description Refs #19831 (plan, Phase 2 sections A and C, and Phase 3 E), #19838 (inventory: "Remote workspace writer lane", and the "Released only by TTL/age" list: "Remote writer lane: 60 min"). **What happened?** `WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS` (60 min of output silence) decided whether a running holder still held a workspace. This applied in four places: - the remote reservation deferral; - the parked busy-workspace wait; - `findSharedWorkspaceHolder`, which stopped counting the holder at all; - `findExecutionWorkspaceHolder`, which flagged it stale. **Expected behavior** A running holder whose controller lease is held is live by fence. Only a holder with no held fence is judged by the silence cutoff, which keeps the visible failure instead of parking forever. The check is database-only (`runOwnerFenceHeldSql`): no process probe on the workspace-preparation path before a turn starts. ## What Changed - `remote-workspace-writer-hold.ts`: new `runOwnerFenceHeldSql`. It is true while `heartbeat_runs.controller_lease_expires_at > now()` or the run's native coordinator lease is held. `RemoteWorkspaceWriter.ownerFenceHeld` is reported for a running writer. - `heartbeat/workspace-preparation.ts`: the reservation keeps deferring behind a silent writer whose fence is held. - `run-dispatch/adapters/workspace-busy-wait.ts`: a parked wait behind a running holder with a held fence stays parked and re-checks at the parked cadence. - `heartbeat/retries.ts`: `findSharedWorkspaceHolder` counts a silent holder whose fence is held. `findExecutionWorkspaceHolder` flags a holder stale only when it is silent and its fence lapsed. ## Verification - `heartbeat/workspace-preparation.test.ts`: 138 pass. The extended "waits for the live writer of an inherited remote … at reservation" (both modes) fails on `master` and passes here: a silent writer with a held lease defers, and once the lease lapses it is the 409 as before. - `heartbeat-workspace-busy.test.ts`: 41 pass. The new "still defers behind a silent holder whose controller renews its lease" fails on `master` and passes here. The existing "does not defer when the only holder has been silent past the staleness threshold" (no lease) still passes. - The server package typecheck passes. ## Risks - A silent holder whose controller is alive now blocks waiters for as long as it holds its lease. That is the safe direction. A hung but live run is the active-run silence watchdog's to handle, and a dead controller's lease lapses within a minute. - The extra predicate is an indexed lookup of the run's own row and its coordinator row. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../src/__tests__/heartbeat-workspace-busy.test.ts | 12 ++++++++++++ .../run-dispatch/adapters/workspace-busy-wait.ts | 6 ++++-- server/src/services/heartbeat/retries.ts | 6 ++++-- .../heartbeat/workspace-preparation.test.ts | 14 ++++++++++++++ .../services/heartbeat/workspace-preparation.ts | 3 ++- .../src/services/remote-workspace-writer-hold.ts | 13 ++++++++++--- 6 files changed, 46 insertions(+), 8 deletions(-) diff --git a/server/src/__tests__/heartbeat-workspace-busy.test.ts b/server/src/__tests__/heartbeat-workspace-busy.test.ts index f21701617da..8f055b90ce0 100644 --- a/server/src/__tests__/heartbeat-workspace-busy.test.ts +++ b/server/src/__tests__/heartbeat-workspace-busy.test.ts @@ -1303,6 +1303,18 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => { expect(retryRuns).toBe(0); }); + it("still defers behind a silent holder whose controller renews its lease", async () => { + const fixture = await seedWorkspaceFixture({ + holderActivityAt: new Date(Date.now() - WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS - 60_000), + }); + await db.update(heartbeatRuns).set({ controllerLeaseExpiresAt: new Date(Date.now() + 10 * 60_000) }) + .where(eq(heartbeatRuns.id, fixture.holderRunId)); + const run = await heartbeat.invoke(fixture.agentId, "assignment", { issueId: fixture.issueId, wakeReason: "issue_assigned" }, "system"); + const finishedRun = await waitForRunToLeaveActiveStates(run!.id); + expect(finishedRun?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE); + expect(executedRunIds).not.toContain(run!.id); + }); + it("does not defer when the only holder has been silent past the staleness threshold", async () => { const fixture = await seedWorkspaceFixture({ holderActivityAt: new Date(Date.now() - WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS - 60_000), diff --git a/server/src/modules/run-dispatch/adapters/workspace-busy-wait.ts b/server/src/modules/run-dispatch/adapters/workspace-busy-wait.ts index 95553a097f1..96b5548e87f 100644 --- a/server/src/modules/run-dispatch/adapters/workspace-busy-wait.ts +++ b/server/src/modules/run-dispatch/adapters/workspace-busy-wait.ts @@ -3,7 +3,7 @@ import { executionWorkspaces, heartbeatRuns, issues, type Db } from "@paperclipa import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import { notifyScheduledRetryDispatch } from "../../../services/run-dispatch-notifications.js"; import { notifyScheduledRetryCancellation } from "../../../services/scheduled-retry-cancellation-notifications.js"; -import { readRemoteWorkspaceWriterHold } from "../../../services/remote-workspace-writer-hold.js"; +import { readRemoteWorkspaceWriterHold, runOwnerFenceHeldSql } from "../../../services/remote-workspace-writer-hold.js"; import { WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, WORKSPACE_BUSY_PARKED_RECHECK_MS, @@ -42,6 +42,7 @@ export async function parkWorkspaceBusyRetryInTx(tx: Db, run: HeartbeatRun, now: .select({ status: heartbeatRuns.status, lastActivityAt: sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt})`, + ownerFenceHeld: runOwnerFenceHeldSql(), }) .from(heartbeatRuns) .where(and(eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.id, wait.holderRunId))); @@ -50,7 +51,8 @@ export async function parkWorkspaceBusyRetryInTx(tx: Db, run: HeartbeatRun, now: // Reservation defers on that hold, so the wait stays parked through it too. const settlingSince = holder && holder.status !== "running" ? await settlingWriterActivityAt(tx, run, wait) : null; const holderStaleAt = holder?.status === "running" - ? new Date(holder.lastActivityAt).getTime() + WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS + ? Math.max(new Date(holder.lastActivityAt).getTime() + WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, + holder.ownerFenceHeld === true ? now.getTime() + WORKSPACE_BUSY_PARKED_RECHECK_MS : 0) : settlingSince ? settlingSince.getTime() + WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS : null; const holderLive = holderStaleAt !== null && holderStaleAt > now.getTime(); diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 7e5a51ea8e5..4308d403e98 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1,3 +1,4 @@ +import { runOwnerFenceHeldSql } from "../remote-workspace-writer-hold.js"; import { isInterruptedNativePreparation } from "../cancelled-native-startup.js"; import { isPreDispatchExecutionWaitVerified } from "../pre-dispatch-execution-wait.js"; import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; @@ -1440,7 +1441,8 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen ne(heartbeatRuns.id, input.excludeRunId), // Last observed activity: output beats start beats creation. A run // that started recently but has not written output yet is live. - sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) >= ${staleCutoff.toISOString()}::timestamptz`, + or(sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) >= ${staleCutoff.toISOString()}::timestamptz`, + runOwnerFenceHeldSql()), eq(issues.projectWorkspaceId, input.projectWorkspaceId), ne(sql`${issues.id}::text`, input.excludeIssueId), ...(input.honorIsolatedWorkspaceModes @@ -1480,7 +1482,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen (input.now ?? new Date()).getTime() - WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, ); - const lastActivityBefore = sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) < ${staleCutoff.toISOString()}::timestamptz`; + const lastActivityBefore = sql`(coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) < ${staleCutoff.toISOString()}::timestamptz and not ${runOwnerFenceHeldSql()})`; return await db .select({ runId: heartbeatRuns.id, diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index 802934c00df..487e4a09a5b 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -516,6 +516,20 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace await db.update(heartbeatRuns).set({ lastOutputAt: new Date(Date.now() - WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS - 1_000) }) .where(eq(heartbeatRuns.id, writer.id)); + await db.insert(nativeRunFinalizations).values({ runId: writer.id, companyId: parent.companyId, issueId: parent.id, + phase: "observed", attempt: 1, controllerGeneration: 1, leaseOwner: "silent-native-controller", + leaseExpiresAt: new Date(Date.now() + 60_000) }); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ + constructor: WorkspaceBusyDeferral, code: "workspace_busy", holder: { runId: writer.id } }); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }) + .where(eq(nativeRunFinalizations.runId, writer.id)); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ status: 409 }); + await db.update(heartbeatRuns).set({ controllerLeaseExpiresAt: new Date(Date.now() + 60_000) }) + .where(eq(heartbeatRuns.id, writer.id)); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ + constructor: WorkspaceBusyDeferral, code: "workspace_busy", holder: { runId: writer.id } }); + await db.update(heartbeatRuns).set({ controllerLeaseExpiresAt: new Date(Date.now() - 1_000) }) + .where(eq(heartbeatRuns.id, writer.id)); await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ status: 409 }); // An ended writer whose save has not settled still holds the workspace: // the next run waits from the writer's end, within the same holder cutoff. diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index d42489f5649..f34e4cdd289 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -1056,7 +1056,8 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace } : {}), }).catch(error => { if (!(error instanceof RemoteWorkspaceWriterActiveError) - || error.writer.lastActivityAt.getTime() < Date.now() - WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS) throw error; + || (!error.writer.ownerFenceHeld + && error.writer.lastActivityAt.getTime() < Date.now() - WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS)) throw error; throw new WorkspaceBusyDeferral({ holder: error.writer, executionWorkspaceId: error.workspaceId, deferralAttempt: run.scheduledRetryReason === WORKSPACE_BUSY_RETRY_REASON ? run.scheduledRetryAttempt ?? 0 : 0, wasIssueAssignee: input.task.wasIssueAssignee, holderEnded: error.writer.ended === true }); diff --git a/server/src/services/remote-workspace-writer-hold.ts b/server/src/services/remote-workspace-writer-hold.ts index 6fa682ab7c3..721239f9777 100644 --- a/server/src/services/remote-workspace-writer-hold.ts +++ b/server/src/services/remote-workspace-writer-hold.ts @@ -1,5 +1,5 @@ import { and, eq, sql } from "drizzle-orm"; -import { environmentLeases, heartbeatRuns, issueRecoveryActions, issues, type Db } from "@paperclipai/db"; +import { environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; import type { RemoteWorkspaceAuthority } from "@paperclipai/shared"; import { hasRecoveredCleanupStopReceipt, hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { canonicalExecutionUuid } from "./execution-identity.js"; @@ -12,8 +12,15 @@ export type RemoteWorkspaceWriter = { runId: string; agentId: string; issueId: string; issueIdentifier: string | null; lastActivityAt: Date; /** The writer run ended; its workspace save or sandbox release has not settled. */ ended?: boolean; + ownerFenceHeld?: boolean; }; +export function runOwnerFenceHeldSql() { + return sql`(coalesce(${heartbeatRuns.controllerLeaseExpiresAt} > now(), false) or exists ( + select 1 from ${nativeRunFinalizations} where ${nativeRunFinalizations.runId} = ${heartbeatRuns.id} + and ${nativeRunFinalizations.leaseOwner} is not null and ${nativeRunFinalizations.leaseExpiresAt} > now()))`; +} + /** * Who still holds a remote workspace through its recorded writer run, other * than `runId`. A running writer holds it. An ended writer keeps holding it @@ -33,7 +40,7 @@ export async function readRemoteWorkspaceWriterHold(tx: Db, input: { errorCode: heartbeatRuns.errorCode, executionRecovery: sql`${heartbeatRuns.resultJson}->'executionRecovery'`, contextIssueId: sql`coalesce(${heartbeatRuns.contextSnapshot}->>'issueId', ${heartbeatRuns.contextSnapshot}->>'taskId')`, lastOutputAt: heartbeatRuns.lastOutputAt, startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt, - finishedAt: heartbeatRuns.finishedAt, + finishedAt: heartbeatRuns.finishedAt, ownerFenceHeld: runOwnerFenceHeldSql(), }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, authority.writerRunId))); const writerIssueId = prior ? prior.nativeIssueId ?? prior.issueId ?? prior.contextIssueId : null; @@ -41,7 +48,7 @@ export async function readRemoteWorkspaceWriterHold(tx: Db, input: { .where(and(eq(issues.companyId, companyId), eq(issues.id, canonicalExecutionUuid(sql`${writerIssueId}::text`)))) : []; if (prior?.status === "running" && writerIssue) return { kind: "held", writer: { runId: authority.writerRunId, agentId: prior.agentId, issueId: writerIssue.id, issueIdentifier: writerIssue.identifier, - lastActivityAt: prior.lastOutputAt ?? prior.startedAt ?? prior.createdAt, + lastActivityAt: prior.lastOutputAt ?? prior.startedAt ?? prior.createdAt, ownerFenceHeld: prior.ownerFenceHeld === true, } }; const leases = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, authority.writerRunId)));