diff --git a/doc/experimental/issue-worktree-support.md b/doc/experimental/issue-worktree-support.md index e52e620b6ce..19614a3c36f 100644 --- a/doc/experimental/issue-worktree-support.md +++ b/doc/experimental/issue-worktree-support.md @@ -33,8 +33,10 @@ Explicit project policies and issue workspace settings still take precedence. A Projects and individual issues can set `sharedWorkspaceConcurrency` in their execution workspace policy/settings: - `auto` (the default when absent): allow concurrent shared-workspace runs on `local` and `ssh` environments, and serialize runs on `sandbox` and `plugin` environments. An instance forced to Kubernetes always serializes in `auto` mode. -- `serialize`: on `sandbox` and `plugin` environments, defer a run while another live run holds the same project workspace, using the `workspace_busy` retry path. Local and SSH folders remain concurrent, even when an existing project or issue policy requests serialization. -- `allow`: dispatch alongside a live holder on every environment. +- `serialize`: on `sandbox` and `plugin` environments, defer a run while another live run holds the same project workspace or the same reused shared execution workspace, using the `workspace_busy` retry path. Local and SSH folders remain concurrent, even when an existing project or issue policy requests serialization. +- `allow`: dispatch alongside a live holder on `local` and `ssh` environments. Other environments cannot place concurrent agents in one physical shared folder, so a run with a live holder fails with `configuration_incomplete` (`shared_realization_unsupported`) instead of waiting. + +A reused shared execution workspace follows the same rules. For example, a child task can inherit the shared execution workspace of a parent whose run is still live. In `auto` or `serialize` mode on a sandbox, the child waits for the parent run to finish and then continues automatically. A parent run that is still running but has produced no output for more than an hour still holds the folder. In that case the child fails with `configuration_incomplete` (`shared_realization_unsupported`). It does not share the folder with the parent, and it does not wait without limit. Issue settings override the project policy, which overrides the default `auto`. The final execution environment determines whether workspace serialization applies; an instance forced to Kubernetes is a sandbox even if the agent normally runs locally. Local and SSH folder runs never wait for exclusive workspace ownership. Task checkout and per-agent concurrency limits still apply. When concurrency is allowed and a live holder exists, Paperclip adds the holder run and issue to the dispatched task context so agents can coordinate concurrent mutations through commits. The setting is optional JSON policy data, so existing databases require no migration. diff --git a/server/src/__tests__/heartbeat-workspace-busy.test.ts b/server/src/__tests__/heartbeat-workspace-busy.test.ts index 461d2beaa90..55649bc9ba8 100644 --- a/server/src/__tests__/heartbeat-workspace-busy.test.ts +++ b/server/src/__tests__/heartbeat-workspace-busy.test.ts @@ -252,9 +252,13 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => { agentId: string; issueId: string; nonAssigneeAgentId: string; + sharedExecutionWorkspaceId: string | null; } async function seedWorkspaceFixture(input?: { + // Bind both issues to one reused shared execution workspace, as when a child + // task inherits the workspace of a parent whose run is still live. + sharedExecutionWorkspace?: boolean; holderIssueWorkspaceSettings?: Record | null; holderProjectWorkspaceId?: string; holderActivityAt?: Date; @@ -272,6 +276,7 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => { const agentId = randomUUID(); const issueId = randomUUID(); const nonAssigneeAgentId = randomUUID(); + const sharedExecutionWorkspaceId = input?.sharedExecutionWorkspace ? randomUUID() : null; // Deferral/retry tests exercise sandbox protection. Local and SSH folders // must remain concurrent even when an older policy requests serialization. const agentEnvironmentDriver = input?.agentEnvironmentDriver ?? "sandbox"; @@ -361,6 +366,21 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => { }); } + if (sharedExecutionWorkspaceId) { + await db.insert(executionWorkspaces).values({ + id: sharedExecutionWorkspaceId, + companyId, + projectId, + mode: "shared_workspace", + strategyType: "project_primary", + name: "Inherited parent workspace", + status: "active", + cwd: workspaceCwd, + providerType: "local_fs", + providerRef: workspaceCwd, + }); + } + const holderActivityAt = input?.holderActivityAt ?? now; await db.insert(heartbeatRuns).values({ id: holderRunId, @@ -394,6 +414,7 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => { executionLockedAt: now, issueNumber: 1, identifier: `${issuePrefix}-1`, + ...(sharedExecutionWorkspaceId ? { executionWorkspaceId: sharedExecutionWorkspaceId } : {}), ...(input?.holderIssueWorkspaceSettings !== undefined ? { executionWorkspaceSettings: input.holderIssueWorkspaceSettings } : {}), @@ -411,6 +432,9 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => { projectWorkspaceId, issueNumber: 2, identifier: `${issuePrefix}-2`, + ...(sharedExecutionWorkspaceId + ? { executionWorkspaceId: sharedExecutionWorkspaceId, executionWorkspacePreference: "reuse_existing" } + : {}), executionWorkspaceSettings: input?.issueWorkspaceSettings === undefined ? { sharedWorkspaceConcurrency: "serialize" } @@ -427,6 +451,7 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => { agentId, issueId, nonAssigneeAgentId, + sharedExecutionWorkspaceId, }; } @@ -647,6 +672,140 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => { expect(retryRuns).toHaveLength(0); }); + describe("reused shared execution workspace in a sandbox", () => { + // The holder uses a different project workspace so only the shared execution + // workspace connects the two runs. + const seedInheritedWorkspace = (input?: Parameters[0]) => seedWorkspaceFixture({ + sharedExecutionWorkspace: true, + holderProjectWorkspaceId: randomUUID(), + issueWorkspaceSettings: { mode: "shared_workspace" }, + ...input, + }); + + it("defers instead of failing while a live run holds the workspace", async () => { + const fixture = await seedInheritedWorkspace(); + + const run = await heartbeat.invoke( + fixture.agentId, + "assignment", + { issueId: fixture.issueId, wakeReason: "issue_assigned" }, + "system", + ); + expect(run).not.toBeNull(); + + const finishedRun = await waitForRunToLeaveActiveStates(run!.id); + expect(finishedRun?.status).toBe("cancelled"); + expect(finishedRun?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE); + expect(finishedRun?.error).not.toContain("cannot place concurrent agents in one physical shared folder"); + expect(finishedRun?.resultJson).toMatchObject({ + executionRecovery: { kind: "workspace_wait", providerWorkStarted: false }, + cancellation: { reason: "Waiting for the shared execution workspace" }, + workspaceBusy: { + projectWorkspaceId: null, + executionWorkspaceId: fixture.sharedExecutionWorkspaceId, + holderRunId: fixture.holderRunId, + holderIssueId: fixture.holderIssueId, + deferralAttempt: 0, + }, + }); + expect(executedRunIds).not.toContain(run!.id); + expect(await waitForRetryRun(run!.id)).toMatchObject({ + status: "scheduled_retry", + scheduledRetryReason: WORKSPACE_BUSY_RETRY_REASON, + }); + await waitForPendingRunFailureReports(); + expect(captureRunFailure.mock.calls.filter(([report]) => report.runId === run!.id)).toHaveLength(0); + const [issueRow] = await db.select({ status: issues.status }).from(issues).where(eq(issues.id, fixture.issueId)); + expect(issueRow?.status).not.toBe("blocked"); + }); + + it("resumes in the inherited workspace once the holder finishes", async () => { + const fixture = await seedInheritedWorkspace(); + + const run = await heartbeat.invoke( + fixture.agentId, + "assignment", + { issueId: fixture.issueId, wakeReason: "issue_assigned" }, + "system", + ); + expect(run).not.toBeNull(); + expect((await waitForRunToLeaveActiveStates(run!.id))?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE); + const retryRun = await waitForRetryRun(run!.id); + expect(retryRun?.status).toBe("scheduled_retry"); + + await db + .update(heartbeatRuns) + .set({ status: "succeeded", finishedAt: new Date() }) + .where(eq(heartbeatRuns.id, fixture.holderRunId)); + const afterDue = new Date(new Date(retryRun!.scheduledRetryAt!).getTime() + 1_000); + expect((await heartbeat.promoteDueScheduledRetries(afterDue)).runIds).toContain(retryRun!.id); + await heartbeat.resumeQueuedRuns(); + + expect((await waitForRunToLeaveActiveStates(retryRun!.id))?.status).toBe("succeeded"); + expect(executedRunIds).toContain(retryRun!.id); + expect(executedInputs.get(retryRun!.id)?.context.paperclipWorkspace).toMatchObject({ + cwd: workspaceCwd, + mode: "shared_workspace", + }); + const [issueRow] = await db + .select({ executionWorkspaceId: issues.executionWorkspaceId }) + .from(issues) + .where(eq(issues.id, fixture.issueId)); + expect(issueRow?.executionWorkspaceId).toBe(fixture.sharedExecutionWorkspaceId); + }); + + it("still rejects an explicit allow policy that requires concurrent physical sharing", async () => { + const fixture = await seedInheritedWorkspace({ + issueWorkspaceSettings: { mode: "shared_workspace", sharedWorkspaceConcurrency: "allow" }, + }); + + const run = await heartbeat.invoke( + fixture.agentId, + "assignment", + { issueId: fixture.issueId, wakeReason: "issue_assigned" }, + "system", + ); + expect(run).not.toBeNull(); + + const finishedRun = await waitForRunToLeaveActiveStates(run!.id); + expect(finishedRun?.status).toBe("failed"); + expect(finishedRun?.error).toContain("cannot place concurrent agents in one physical shared folder"); + expect(executedRunIds).not.toContain(run!.id); + expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, fixture.companyId), + eq(heartbeatRuns.scheduledRetryReason, WORKSPACE_BUSY_RETRY_REASON), + ))).toHaveLength(0); + }); + + it("neither overlaps nor waits forever on a quiet holder that is still running", async () => { + const fixture = await seedInheritedWorkspace({ + holderActivityAt: new Date(Date.now() - WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS - 60_000), + }); + + const run = await heartbeat.invoke( + fixture.agentId, + "assignment", + { issueId: fixture.issueId, wakeReason: "issue_assigned" }, + "system", + ); + expect(run).not.toBeNull(); + + const finishedRun = await waitForRunToLeaveActiveStates(run!.id); + expect(finishedRun?.status).toBe("failed"); + expect(finishedRun?.errorCode).not.toBe(WORKSPACE_BUSY_ERROR_CODE); + expect(finishedRun?.error).toContain(`run ${fixture.holderRunId} still holds this shared execution workspace`); + expect(finishedRun?.resultJson).toMatchObject({ + configurationIncomplete: { reason: "shared_realization_unsupported", holderRunId: fixture.holderRunId, holderStale: true }, + }); + expect(executedRunIds).not.toContain(run!.id); + expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, fixture.companyId), + eq(heartbeatRuns.scheduledRetryReason, WORKSPACE_BUSY_RETRY_REASON), + ))).toHaveLength(0); + expect((await heartbeat.getRun(fixture.holderRunId))?.status).toBe("running"); + }); + }); + it.each([0, 35 * 60_000])("defers a run in a busy shared workspace after %i ms of holder silence", async (silenceMs) => { const fixture = await seedWorkspaceFixture({ holderActivityAt: new Date(Date.now() - silenceMs) }); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 1d17cbc8cd8..f8bbb4b6330 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -953,6 +953,7 @@ export function heartbeatService( timerClaimWasFirstHeartbeat, scheduleInteractionContinuationInfrastructureRetryIfEligible, findSharedWorkspaceHolder, + findExecutionWorkspaceHolder, finalizeAiConnectionBusyDeferral, finalizeWorkspaceBusyDeferral, promoteDueScheduledRetries, @@ -2624,7 +2625,7 @@ export function heartbeatService( projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings, }, config: { config, sourceWorkspaceConfig, issueAssigneeOverrides }, - services: { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder }, + services: { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder, findExecutionWorkspaceHolder }, })); let nativeChatExpectedCwd = selectedNativeChatExpectedCwd; const executionConfiguration = await timePreparation("execution_configuration", () => prepareHeartbeatExecutionConfiguration(db, { diff --git a/server/src/services/heartbeat/environment-selection.test.ts b/server/src/services/heartbeat/environment-selection.test.ts index 884ddf1d1dc..8b83110fe89 100644 --- a/server/src/services/heartbeat/environment-selection.test.ts +++ b/server/src/services/heartbeat/environment-selection.test.ts @@ -53,7 +53,7 @@ describe.skipIf(!support.supported)("heartbeat environment selection boundary", projectExecutionWorkspacePolicy: null, issueExecutionWorkspaceSettings: null, }, config: { config: {}, sourceWorkspaceConfig: {}, issueAssigneeOverrides: null }, - services: { environmentsSvc: environmentService(db), executionWorkspacesSvc: executionWorkspaceService(db), instanceSettings: instanceSettingsService(db), findSharedWorkspaceHolder: vi.fn(async () => null) }, + services: { environmentsSvc: environmentService(db), executionWorkspacesSvc: executionWorkspaceService(db), instanceSettings: instanceSettingsService(db), findSharedWorkspaceHolder: vi.fn(async () => null), findExecutionWorkspaceHolder: vi.fn(async () => null) }, }; const workspace = async () => { const created = await executionWorkspaceService(db).create({ companyId: company.id, projectId: project.id, sourceIssueId: issue.id, name: "Saved workspace", mode: "isolated_workspace", strategyType: "git_worktree", cwd: home, providerRef: home }); diff --git a/server/src/services/heartbeat/environment-selection.ts b/server/src/services/heartbeat/environment-selection.ts index 8a732a90dba..32cf9b05d78 100644 --- a/server/src/services/heartbeat/environment-selection.ts +++ b/server/src/services/heartbeat/environment-selection.ts @@ -1,5 +1,4 @@ import { heartbeatRuns, issues, type Db, type agents } from "@paperclipai/db"; -import { and, eq, ne, sql } from "drizzle-orm"; import type { ExecutionWorkspace, IssueExecutionWorkspaceSettings, ProjectExecutionWorkspacePolicy } from "@paperclipai/shared"; import { parseObject } from "../../adapters/utils.js"; import { logger } from "../../middleware/logger.js"; @@ -69,6 +68,7 @@ export interface HeartbeatEnvironmentSelectionInput { "ensureLocalEnvironment" | "findManagedSandboxEnvironment" | "findKubernetesEnvironment" | "ensureKubernetesEnvironment" | "getById">; instanceSettings: Pick, "get" | "getExperimental">; findSharedWorkspaceHolder: ReturnType["findSharedWorkspaceHolder"]; + findExecutionWorkspaceHolder: ReturnType["findExecutionWorkspaceHolder"]; }; } @@ -84,7 +84,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings, } = input.policy; const { config, sourceWorkspaceConfig, issueAssigneeOverrides } = input.config; - const { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder } = input.services; + const { executionWorkspacesSvc, environmentsSvc, instanceSettings, findSharedWorkspaceHolder, findExecutionWorkspaceHolder } = input.services; // A native run's execution input is immutable once persisted. Recovery must therefore // restore the workspace bound to that input rather than consulting the issue's current // workspace pointer: a newer run may already have moved or cleared the issue binding while @@ -307,23 +307,45 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron }, ); } - if (!nativeRecoveryExecutionWorkspaceId && reusableExistingExecutionWorkspace && - effectiveExecutionWorkspaceMode === "shared_workspace" && - selectedEnvironmentForConfig?.driver !== "local" && selectedEnvironmentForConfig?.driver !== "ssh") { - const [remoteSharedHolder] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, agent.companyId), ne(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), - sql`exists (select 1 from ${issues} where ${issues.companyId} = ${agent.companyId} - and ${issues.executionWorkspaceId} = ${reusableExistingExecutionWorkspace.id} - and (${issues.executionRunId} = ${heartbeatRuns.id} or ${issues.id}::text = coalesce(${heartbeatRuns.contextSnapshot}->>'issueId', ${heartbeatRuns.contextSnapshot}->>'taskId')))`, - )).limit(1); - if (remoteSharedHolder) throw new ConfigurationIncompleteFailure("This environment cannot place concurrent agents in one physical shared folder. Use local/SSH sharing or isolated task workspaces.", { - configurationIncomplete: { reason: "shared_realization_unsupported", issueId }, - }); - } const sharedWorkspaceConcurrency = resolveSharedWorkspaceConcurrency({ projectPolicy: projectExecutionWorkspacePolicy, issueSettings: issueExecutionWorkspaceSettings, }); + // A reused shared execution workspace (for example one a child task inherited from a + // parent whose run is still live) follows the same policy as a shared project workspace: + // only an explicit "allow" is unsupported outside local/SSH; otherwise wait for the holder. + // A holder silent past the staleness cutoff is still running, so its folder is not free: + // fail rather than overlap it or wait on it forever. + if (!nativeRecoveryExecutionWorkspaceId && reusableExistingExecutionWorkspace && + effectiveExecutionWorkspaceMode === "shared_workspace" && + selectedEnvironmentForConfig?.driver !== "local" && selectedEnvironmentForConfig?.driver !== "ssh") { + const executionWorkspaceHolder = await findExecutionWorkspaceHolder({ + companyId: agent.companyId, + executionWorkspaceId: reusableExistingExecutionWorkspace.id, + excludeRunId: run.id, + }); + if (executionWorkspaceHolder) { + if (executionWorkspaceHolder.stale) { + throw new ConfigurationIncompleteFailure(`This environment cannot place concurrent agents in one physical shared folder, and run ${executionWorkspaceHolder.runId} still holds this shared execution workspace after more than an hour without output. Use local/SSH sharing or isolated task workspaces.`, { + configurationIncomplete: { reason: "shared_realization_unsupported", issueId, holderRunId: executionWorkspaceHolder.runId, holderStale: true }, + }); + } + if (sharedWorkspaceConcurrency === "allow") { + throw new ConfigurationIncompleteFailure("This environment cannot place concurrent agents in one physical shared folder. Use local/SSH sharing or isolated task workspaces.", { + configurationIncomplete: { reason: "shared_realization_unsupported", issueId }, + }); + } + throw new WorkspaceBusyDeferral({ + holder: executionWorkspaceHolder, + executionWorkspaceId: reusableExistingExecutionWorkspace.id, + deferralAttempt: + run.scheduledRetryReason === WORKSPACE_BUSY_RETRY_REASON + ? (run.scheduledRetryAttempt ?? 0) + : 0, + wasIssueAssignee: issueContext?.assigneeAgentId === agent.id, + }); + } + } // A live holder is always consulted for shared workspaces. Depending on policy and the final // execution target it either remains the existing deferral gate or becomes dispatch context. // Local/SSH folders never take an exclusive workspace lock, including when older diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index d2e7cb0e5b7..eaf07ae7b7f 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -173,31 +173,40 @@ export interface SharedWorkspaceHolder { issueIdentifier: string | null; } +export interface ExecutionWorkspaceHolder extends SharedWorkspaceHolder { + // Silent past WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS but still running. + stale: boolean; +} + // Pre-dispatch gate outcome: another running run currently holds the issue's -// shared project workspace. Not a failure — the run is parked as a bounded -// scheduled retry and re-attempted once the holder finishes, so two agents -// never mutate the same working tree concurrently. +// shared project workspace, or the shared execution workspace it reuses. Not a +// failure — the run is parked as a bounded scheduled retry and re-attempted +// once the holder finishes, so two agents never mutate the same working tree +// concurrently. export class WorkspaceBusyDeferral extends Error { code = WORKSPACE_BUSY_ERROR_CODE; holder: SharedWorkspaceHolder; - projectWorkspaceId: string; + projectWorkspaceId: string | null; + executionWorkspaceId: string | null; deferralAttempt: number; wasIssueAssignee: boolean; constructor(input: { holder: SharedWorkspaceHolder; - projectWorkspaceId: string; + projectWorkspaceId?: string | null; + executionWorkspaceId?: string | null; deferralAttempt: number; wasIssueAssignee: boolean; }) { super( - `Shared project workspace is busy: run ${input.holder.runId} (issue ${ + `Shared ${input.executionWorkspaceId ? "execution" : "project"} workspace is busy: run ${input.holder.runId} (issue ${ input.holder.issueIdentifier ?? input.holder.issueId }) is still running`, ); this.name = "WorkspaceBusyDeferral"; this.holder = input.holder; - this.projectWorkspaceId = input.projectWorkspaceId; + this.projectWorkspaceId = input.projectWorkspaceId ?? null; + this.executionWorkspaceId = input.executionWorkspaceId ?? null; this.deferralAttempt = input.deferralAttempt; this.wasIssueAssignee = input.wasIssueAssignee; } @@ -1365,6 +1374,56 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen .then((rows) => rows[0] ?? null); } + // Finds a running run (other than the caller's) on an issue bound to the + // same reused execution workspace, typically a parent run that is still + // active after delegating a child that inherited its workspace. Unlike + // findSharedWorkspaceHolder, a holder silent past + // WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS is still returned (flagged stale): + // silence does not prove the holder stopped, so the caller must not treat + // the folder as free. A live holder is preferred over a stale one. + async function findExecutionWorkspaceHolder(input: { + companyId: string; + executionWorkspaceId: string; + excludeRunId: string; + now?: Date; + }): Promise { + const staleCutoff = new Date( + (input.now ?? new Date()).getTime() - + WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, + ); + const lastActivityBefore = sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) < ${staleCutoff.toISOString()}::timestamptz`; + return await db + .select({ + runId: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + issueId: sql`${issues.id}::text`, + issueIdentifier: issues.identifier, + stale: lastActivityBefore, + }) + .from(heartbeatRuns) + .innerJoin( + issues, + and( + eq(issues.companyId, heartbeatRuns.companyId), + eq(issues.executionWorkspaceId, input.executionWorkspaceId), + or( + eq(issues.executionRunId, heartbeatRuns.id), + sql`${issues.id}::text = coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId')`, + ), + ), + ) + .where( + and( + eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.status, "running"), + ne(heartbeatRuns.id, input.excludeRunId), + ), + ) + .orderBy(asc(lastActivityBefore), asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + } + // Credential rotation can briefly contend with a fresh runtime read. Keep // the task on its automatic pre-provider retry path while the lock clears. async function finalizeAiConnectionBusyDeferral( @@ -1446,11 +1505,12 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen source: "control_plane", expected: true, initiator: { type: "system" }, - reason: "Waiting for the shared project workspace", + reason: `Waiting for the shared ${deferral.executionWorkspaceId ? "execution" : "project"} workspace`, recordedAt: now.toISOString(), }, workspaceBusy: { projectWorkspaceId: deferral.projectWorkspaceId, + executionWorkspaceId: deferral.executionWorkspaceId, holderRunId: deferral.holder.runId, holderIssueId: deferral.holder.issueId, deferralAttempt: deferral.deferralAttempt, @@ -1514,6 +1574,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen : `Deferred: ${deferral.message}. No retry could be scheduled; releasing the issue for other runs.`, payload: { projectWorkspaceId: deferral.projectWorkspaceId, + executionWorkspaceId: deferral.executionWorkspaceId, holderRunId: deferral.holder.runId, holderIssueId: deferral.holder.issueId, deferralAttempt: deferral.deferralAttempt, @@ -1827,6 +1888,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen timerClaimWasFirstHeartbeat, scheduleInteractionContinuationInfrastructureRetryIfEligible, findSharedWorkspaceHolder, + findExecutionWorkspaceHolder, finalizeAiConnectionBusyDeferral, finalizeWorkspaceBusyDeferral, promoteDueScheduledRetries, diff --git a/server/src/services/recovery/stranded-notice.test.ts b/server/src/services/recovery/stranded-notice.test.ts index c824d684ad2..2ad1e7f45d5 100644 --- a/server/src/services/recovery/stranded-notice.test.ts +++ b/server/src/services/recovery/stranded-notice.test.ts @@ -60,6 +60,22 @@ describe("stranded recovery notice seeds", () => { expect(disabled.body).toContain("an operator disabled it"); }); + it("explains unsupported concurrent shared workspace placement instead of missing secrets", () => { + const seed = buildConfigurationIncompleteRecoveryNoticeSeed({ reason: "shared_realization_unsupported" }); + expect(seed.title).toBe("Shared workspace unavailable"); + expect(seed.body).toContain("cannot place concurrent agents in one physical shared folder"); + expect(seed.body).not.toContain("secret/env bindings"); + expect(seed.nextAction).toContain("isolated task workspace"); + }); + + it("explains a quiet holder that still owns the shared workspace", () => { + const seed = buildConfigurationIncompleteRecoveryNoticeSeed({ reason: "shared_realization_unsupported", holderStale: true }); + expect(seed.title).toBe("Shared workspace still held"); + expect(seed.body).toContain("no output for more than an hour"); + expect(seed.body).not.toContain("secret/env bindings"); + expect(seed.nextAction).toContain("stop it if it is stuck"); + }); + it("keeps the secret-binding copy for other configuration gaps", () => { expect(buildConfigurationIncompleteRecoveryNoticeSeed({ reason: "secret_binding_missing" }).body).toContain( "secret/env bindings", diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index 41ec1a08d9e..dfe2a71b80f 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -121,6 +121,22 @@ export function buildConfigurationIncompleteRecoveryNoticeSeed( tone: "danger", }; } + if (readNonEmptyStringField(configurationIncomplete, "reason") === "shared_realization_unsupported") { + if (configurationIncomplete?.holderStale === true) { + return { + title: "Shared workspace still held", + body: "Paperclip stopped before the agent started because another run still holds this task's shared workspace but has produced no output for more than an hour, and this execution environment cannot place concurrent agents in one physical shared folder.", + nextAction: "Check whether the other run is still working and stop it if it is stuck, or select an isolated task workspace, then retry the task.", + tone: "danger", + }; + } + return { + title: "Shared workspace unavailable", + body: "Paperclip stopped before the agent started because this task allows concurrent agents in a shared workspace that another live run is using, and this execution environment cannot place concurrent agents in one physical shared folder.", + nextAction: "Select an isolated task workspace, use a local or SSH environment, or change shared workspace concurrency from `allow` so the task waits for the other run.", + tone: "danger", + }; + } if (readNonEmptyStringField(configurationIncomplete, "reason") === "ai_connection_unavailable") { return { title: "AI connection needs attention",