From 1c33275ada8540a72e1c5ec2b159cc732c24f714 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 12 Nov 2026 20:36:05 -0600 Subject: [PATCH] fix(heartbeat): start the new owner after reassignment away from a queued creator run (#21256) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - A lead agent often creates a task for itself and then reassigns it to another agent > - The creator is busy in its own chat run, so the new task's assignment run stays queued and does not take the task's execution lock > - #21030 releases the outgoing assignee's never-started run on reassignment, but only when that run holds the execution lock > - Here the run did not hold the lock, so it stayed queued. The new owner's wake then adopted it as the active run and was deferred behind it > - When the queue later withheld the stale run as `issue_assignee_changed`, nothing promoted the deferred wake, and the stranded-wake sweep ignores assignment wakes. The new owner never ran > - This pull request closes all three gaps > - The benefit is that a reassigned task starts its new owner at once, and recovery still starts it if a release is missed ## Linked Issues or Issue Description Refs #21030 **What happened?** In the paid chat evals `onboarding.first-work` and `questions-and-approvals.duplicate-answer` (OpenCode with `openrouter/deepseek/deepseek-v4.1-flash-1017`, master b1ef1f3022), the CEO created a task for itself while its chat run was active, then reassigned it to Morgan. The CEO's queued assignment run was withheld at the execution gate with "issue assignee changed before the queued run could start; the new owner will be woken instead". Morgan's wake stayed `deferred_issue_execution` until teardown, about 4.5 minutes later. Morgan never ran. **Steps to reproduce** 1. Give an agent a running run on another task. 2. Create a task assigned to that agent, so its assignment run is queued without the execution lock. 3. Reassign the task to a second agent. 4. The second agent's wake is deferred behind the stale run and is not promoted when that run is withheld. **Expected behavior** The stale run is released and the new owner gets exactly one run for the task, promptly. ## What Changed - `server/src/routes/issues.ts`: on reassignment, when no live run holds the task's lock, also release the outgoing assignee's never-started `issue_assigned` run for that task. This uses the existing `issue_reassigned` stop from #21030. The run is read through a new `heartbeat.getQueuedAssignmentRunForIssue` accessor in `heartbeat/run-state.ts`; route test mocks of the heartbeat service gain that method. - `server/src/services/heartbeat/queue.ts`: when the queue gate withholds a queued run as `issue_assignee_changed`, it now releases the task and promotes the new owner's deferred wake. The promoted run starts after the claim lock is released (`deferredPostCommitEffects`), like the neighboring cancellation branches. A failed promotion is logged and left to recovery. - `server/src/modules/run-dispatch/adapters/postgres.ts`: the same cancellation hints each deferred wake of the task to the deferred-receipt worker in the same transaction, so recovery runs even if the process stops before promotion. - `server/src/services/heartbeat/queue.ts`, `deferred-receipt-work.ts`, `deferred-execution-notifications.ts`: the stranded-wake sweep now includes deferred `issue_assigned` wakes of the current assignee. It promotes them only when no queued, running or retrying run remains for the task, through the normal release admission. - `server/src/__tests__/heartbeat-reassignment-deferred-wake.test.ts`: the cohort's state through the real wake path. One case covers the gate withholding the stale run. One case covers a wake left deferred after the blocking run finished. Both expect exactly one run by the new owner, executed. - `server/src/__tests__/issue-reassignment-queued-run.test.ts`: the #21030 case now also runs when the queued run does not hold the lock, and expects exactly one new-owner run. ## Verification - `cd server && npx vitest run src/__tests__/heartbeat-reassignment-deferred-wake.test.ts src/__tests__/issue-reassignment-queued-run.test.ts` passes. On master, both heartbeat cases fail (the wake stays `deferred_issue_execution`) and the no-lock route case fails (the stale run stays queued). With only the gate change removed, the gate case fails and the recovery case still passes. - Focused suites: `deferred-execution-receipt-work`, `heartbeat-lock-release-on-reassignment`, `heartbeat-stale-queue-invalidation`, `issue-agent-mutation-ownership-routes`, `src/modules/run-dispatch`, `src/modules/wake-queue` pass. - `cd server && npx tsc --noEmit` passes. ## Risks - Behavior change: reassignment now cancels the outgoing assignee's queued `issue_assigned` run for the task even when it does not hold the lock. The queue gate already cancelled that run later as `issue_assignee_changed`; now the new owner does not wait for it. - The stranded-wake sweep now also reads deferred assignment wakes. It skips any wake whose task still has a live run, so a wake waiting behind a real owner is not promoted early. - Upgrade safety: no schema or saved-state shape changes. Wakes already saved as `deferred_issue_execution` by the previous release are promoted by the sweep after deploy. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../artifact-review-document-routes.test.ts | 1 + ...ard-native-attachments.integration.test.ts | 1 + ...environment-selection-route-guards.test.ts | 1 + .../__tests__/external-object-routes.test.ts | 1 + ...artbeat-reassignment-deferred-wake.test.ts | 151 ++++++++++++++++++ .../issue-activity-events-routes.test.ts | 1 + ...ue-agent-mutation-ownership-routes.test.ts | 1 + ...issue-assignee-invokability-routes.test.ts | 1 + .../__tests__/issue-attachment-routes.test.ts | 1 + .../issue-closed-workspace-routes.test.ts | 1 + .../issue-comment-cancel-routes.test.ts | 1 + .../issue-comment-reopen-routes.test.ts | 1 + .../issue-execution-policy-routes.test.ts | 1 + .../__tests__/issue-feedback-routes.test.ts | 1 + .../issue-reassignment-queued-run.test.ts | 12 +- ...issue-update-comment-wakeup-routes.test.ts | 1 + .../issue-workspace-command-authz.test.ts | 1 + .../modules/run-dispatch/adapters/postgres.ts | 5 +- server/src/routes/issues.ts | 13 +- .../deferred-execution-notifications.ts | 11 ++ server/src/services/heartbeat.ts | 3 + .../heartbeat/deferred-receipt-work.ts | 3 +- server/src/services/heartbeat/queue.ts | 13 +- server/src/services/heartbeat/run-state.ts | 13 ++ 24 files changed, 225 insertions(+), 14 deletions(-) create mode 100644 server/src/__tests__/heartbeat-reassignment-deferred-wake.test.ts diff --git a/server/src/__tests__/artifact-review-document-routes.test.ts b/server/src/__tests__/artifact-review-document-routes.test.ts index 2a34aa2efb0..d926a954405 100644 --- a/server/src/__tests__/artifact-review-document-routes.test.ts +++ b/server/src/__tests__/artifact-review-document-routes.test.ts @@ -81,6 +81,7 @@ function registerRouteMocks() { reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), }), instanceSettingsService: () => ({ diff --git a/server/src/__tests__/board-native-attachments.integration.test.ts b/server/src/__tests__/board-native-attachments.integration.test.ts index b5e5f493720..1374e6261c9 100644 --- a/server/src/__tests__/board-native-attachments.integration.test.ts +++ b/server/src/__tests__/board-native-attachments.integration.test.ts @@ -51,6 +51,7 @@ vi.mock("../services/heartbeat.js", async (importOriginal) => ({ wakeup, getRun: async () => null, getActiveRunForAgent: async () => null, + getQueuedAssignmentRunForIssue: async () => null, reportRunActivity: async () => undefined, }), })); diff --git a/server/src/__tests__/environment-selection-route-guards.test.ts b/server/src/__tests__/environment-selection-route-guards.test.ts index c08a109f8c0..f3e5a34e820 100644 --- a/server/src/__tests__/environment-selection-route-guards.test.ts +++ b/server/src/__tests__/environment-selection-route-guards.test.ts @@ -84,6 +84,7 @@ vi.mock("../services/index.js", () => ({ heartbeatService: () => ({ getRun: vi.fn(), getActiveRunForAgent: vi.fn(), + getQueuedAssignmentRunForIssue: vi.fn(), }), issueApprovalService: () => ({ listApprovalsForIssue: vi.fn(), diff --git a/server/src/__tests__/external-object-routes.test.ts b/server/src/__tests__/external-object-routes.test.ts index 7df975fe682..738b5947b10 100644 --- a/server/src/__tests__/external-object-routes.test.ts +++ b/server/src/__tests__/external-object-routes.test.ts @@ -71,6 +71,7 @@ function registerRouteMocks() { reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), }), issueApprovalService: () => ({}), diff --git a/server/src/__tests__/heartbeat-reassignment-deferred-wake.test.ts b/server/src/__tests__/heartbeat-reassignment-deferred-wake.test.ts new file mode 100644 index 00000000000..f5556975be1 --- /dev/null +++ b/server/src/__tests__/heartbeat-reassignment-deferred-wake.test.ts @@ -0,0 +1,151 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, + agentWakeupRequests, + authUsers, + companies, + companyMemberships, + createDb, + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, + heartbeatRuns, + issues, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js"; +import { truncateTablesWithDeadlockRetry } from "./helpers/truncate-with-deadlock-retry.js"; +import { heartbeatService } from "../services/heartbeat.ts"; +import { runningProcesses, type AdapterExecutionContext, type AdapterExecutionResult } from "../adapters/index.ts"; + +const mockAdapterExecute = vi.hoisted(() => + vi.fn(async (_context: AdapterExecutionContext): Promise => ({ + exitCode: 0, signal: null, timedOut: false, errorMessage: null, + summary: "Reassignment wake test run.", provider: "test", model: "test-model", + })), +); + +vi.mock("../adapters/index.ts", async () => { + const actual = await vi.importActual("../adapters/index.ts"); + return { ...actual, getServerAdapter: vi.fn(() => ({ supportsLocalAgentJwt: false, execute: mockAdapterExecute })) }; +}); + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping reassignment deferred-wake tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +describeEmbeddedPostgres("the new owner's wake after a task is reassigned away from a queued creator run", () => { + let db!: ReturnType; + let heartbeat!: ReturnType; + let tempDb: Awaited> | null = null; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-reassign-deferred-wake-"); + db = createDb(tempDb.connectionString); + heartbeat = heartbeatService(db, { runtimeEnv: { ...process.env, PAPERCLIP_IN_WORKTREE: "false" } }); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + afterEach(async () => { + mockAdapterExecute.mockClear(); + runningProcesses.clear(); + await drainHeartbeatRunsToQuiescence(db, heartbeat); + await truncateTablesWithDeadlockRetry(db, `TRUNCATE TABLE "issues", "heartbeat_run_events", "activity_log", + "heartbeat_runs", "agent_wakeup_requests", "agent_runtime_state", "agents", "companies" RESTART IDENTITY CASCADE`, + { attempts: 10, delayMs: () => 100 }); + await db.delete(authUsers).where(eq(authUsers.id, "responsible-user")); + }); + + afterAll(async () => { + await db?.$client.end(); + await tempDb?.cleanup(); + }); + + async function seedReassignedTaskWithDeferredNewOwner() { + const companyId = randomUUID(); + const ceoId = randomUUID(); + const morganId = randomUUID(); + const issueId = randomUUID(); + const ceoRunId = randomUUID(); + const ceoWakeId = randomUUID(); + await db.insert(companies).values({ + id: companyId, name: "Reassign Co", issuePrefix: `R${companyId.replace(/-/g, "").slice(0, 5).toUpperCase()}`, + defaultResponsibleUserId: "responsible-user", requireBoardApprovalForNewAgents: false, + }); + await db.insert(authUsers).values({ + id: "responsible-user", name: "Fixture user", email: "responsible-user@example.test", + createdAt: new Date(), updatedAt: new Date(), + }).onConflictDoNothing(); + await db.insert(companyMemberships).values({ + companyId, principalType: "user", principalId: "responsible-user", status: "active", membershipRole: "member", + }); + await db.insert(agents).values([ceoId, morganId].map((id, index) => ({ + id, companyId, name: index === 0 ? "CEO" : "Morgan", role: index === 0 ? "ceo" : "engineer", + status: "idle", adapterType: "codex_local", adapterConfig: {}, + runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } }, permissions: {}, + }))); + await db.insert(issues).values({ + id: issueId, companyId, title: "Make a simple website", status: "todo", priority: "medium", + assigneeAgentId: morganId, + }); + await db.insert(agentWakeupRequests).values({ + id: ceoWakeId, companyId, agentId: ceoId, source: "assignment", triggerDetail: "system", + reason: "issue_assigned", payload: { issueId }, status: "queued", + }); + await db.insert(heartbeatRuns).values({ + id: ceoRunId, companyId, agentId: ceoId, invocationSource: "assignment", triggerDetail: "system", + status: "queued", wakeupRequestId: ceoWakeId, contextSnapshot: { issueId, wakeReason: "issue_assigned" }, + }); + await db.update(agentWakeupRequests).set({ runId: ceoRunId }).where(eq(agentWakeupRequests.id, ceoWakeId)); + + await heartbeat.wakeup(morganId, { + source: "assignment", triggerDetail: "system", reason: "issue_assigned", + requestedByActorType: "system", payload: { issueId }, + }); + const [deferred] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, morganId)); + expect(deferred).toMatchObject({ status: "deferred_issue_execution", runId: null }); + return { companyId, ceoId, morganId, issueId, ceoRunId, deferredWakeId: deferred!.id }; + } + + async function expectOneNewOwnerRun(f: { morganId: string; issueId: string; deferredWakeId: string }) { + await vi.waitFor(async () => { + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.deferredWakeId)); + expect(wake!.status).not.toBe("deferred_issue_execution"); + expect(wake!.runId).toBeTruthy(); + const runs = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.agentId, f.morganId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`)); + expect(runs).toHaveLength(1); + expect(runs[0]!.id).toBe(wake!.runId); + expect(mockAdapterExecute.mock.calls.filter(([context]) => context?.runId === wake!.runId), JSON.stringify({ status: runs[0]!.status, errorCode: runs[0]!.errorCode, error: runs[0]!.error })).toHaveLength(1); + }, { timeout: 15_000 }); + } + + it("promotes the new owner's wake when the queue withholds the stale creator run", async () => { + const f = await seedReassignedTaskWithDeferredNewOwner(); + + await heartbeat.resumeQueuedRuns(); + + const [stale] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.ceoRunId)); + expect(stale).toMatchObject({ status: "cancelled", errorCode: "issue_assignee_changed" }); + expect(mockAdapterExecute.mock.calls.filter(([context]) => context?.runId === f.ceoRunId)).toHaveLength(0); + await expectOneNewOwnerRun(f); + }); + + it("recovers a new owner's wake left deferred after the blocking run already finished", async () => { + const f = await seedReassignedTaskWithDeferredNewOwner(); + await db.update(heartbeatRuns).set({ + status: "cancelled", finishedAt: new Date(), errorCode: "issue_assignee_changed", + error: "Cancelled because issue assignee changed before the queued run could start; the new owner will be woken instead", + }).where(eq(heartbeatRuns.id, f.ceoRunId)); + await db.update(issues).set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null }) + .where(eq(issues.id, f.issueId)); + + await heartbeat.resumeQueuedRuns(); + + await expectOneNewOwnerRun(f); + }); +}); diff --git a/server/src/__tests__/issue-activity-events-routes.test.ts b/server/src/__tests__/issue-activity-events-routes.test.ts index 1fc4088c911..14a4cb6e93d 100644 --- a/server/src/__tests__/issue-activity-events-routes.test.ts +++ b/server/src/__tests__/issue-activity-events-routes.test.ts @@ -36,6 +36,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), })); const mockFeedbackService = vi.hoisted(() => ({ diff --git a/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts b/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts index d31598d2555..b7f16117cab 100644 --- a/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts +++ b/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts @@ -125,6 +125,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async (): Promise => null), getActiveRunForAgent: vi.fn(async (): Promise => null), + getQueuedAssignmentRunForIssue: vi.fn(async (): Promise => null), cancelRun: vi.fn(async (): Promise => null), resumeTaskRetry: vi.fn(async () => undefined), })); diff --git a/server/src/__tests__/issue-assignee-invokability-routes.test.ts b/server/src/__tests__/issue-assignee-invokability-routes.test.ts index 45f0b2e88c6..d6a788b8f13 100644 --- a/server/src/__tests__/issue-assignee-invokability-routes.test.ts +++ b/server/src/__tests__/issue-assignee-invokability-routes.test.ts @@ -38,6 +38,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), })); diff --git a/server/src/__tests__/issue-attachment-routes.test.ts b/server/src/__tests__/issue-attachment-routes.test.ts index ed9451ad0c9..6afe4b14f2a 100644 --- a/server/src/__tests__/issue-attachment-routes.test.ts +++ b/server/src/__tests__/issue-attachment-routes.test.ts @@ -67,6 +67,7 @@ function registerRouteMocks() { reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), }), instanceSettingsService: () => ({ diff --git a/server/src/__tests__/issue-closed-workspace-routes.test.ts b/server/src/__tests__/issue-closed-workspace-routes.test.ts index 9d109d1a63f..ca78112128c 100644 --- a/server/src/__tests__/issue-closed-workspace-routes.test.ts +++ b/server/src/__tests__/issue-closed-workspace-routes.test.ts @@ -44,6 +44,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), })); diff --git a/server/src/__tests__/issue-comment-cancel-routes.test.ts b/server/src/__tests__/issue-comment-cancel-routes.test.ts index 410f697e7b5..88fb71596f0 100644 --- a/server/src/__tests__/issue-comment-cancel-routes.test.ts +++ b/server/src/__tests__/issue-comment-cancel-routes.test.ts @@ -23,6 +23,7 @@ const mockAccessService = vi.hoisted(() => ({ const mockHeartbeatService = vi.hoisted(() => ({ getRun: vi.fn(async (): Promise => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), })); const mockAuthoritativeQueueWakes = vi.hoisted(() => [] as Array>); diff --git a/server/src/__tests__/issue-comment-reopen-routes.test.ts b/server/src/__tests__/issue-comment-reopen-routes.test.ts index 1ade5a31de9..1a1f7d52f1b 100644 --- a/server/src/__tests__/issue-comment-reopen-routes.test.ts +++ b/server/src/__tests__/issue-comment-reopen-routes.test.ts @@ -40,6 +40,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async (): Promise => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async (): Promise => null), })); diff --git a/server/src/__tests__/issue-execution-policy-routes.test.ts b/server/src/__tests__/issue-execution-policy-routes.test.ts index 8fffd3761f1..bda24c1dfee 100644 --- a/server/src/__tests__/issue-execution-policy-routes.test.ts +++ b/server/src/__tests__/issue-execution-policy-routes.test.ts @@ -25,6 +25,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), })); diff --git a/server/src/__tests__/issue-feedback-routes.test.ts b/server/src/__tests__/issue-feedback-routes.test.ts index 643956ae223..13ae62a9da5 100644 --- a/server/src/__tests__/issue-feedback-routes.test.ts +++ b/server/src/__tests__/issue-feedback-routes.test.ts @@ -31,6 +31,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), })); const mockInstanceSettingsService = vi.hoisted(() => ({ diff --git a/server/src/__tests__/issue-reassignment-queued-run.test.ts b/server/src/__tests__/issue-reassignment-queued-run.test.ts index 151e9fc5799..4a3b3e4bf20 100644 --- a/server/src/__tests__/issue-reassignment-queued-run.test.ts +++ b/server/src/__tests__/issue-reassignment-queued-run.test.ts @@ -65,7 +65,7 @@ describeEmbeddedPostgres("reassigning a task whose previous assignee's run has n return testApp; } - async function seed() { + async function seed(options: { queuedRunHoldsLock?: boolean } = {}) { const companyId = randomUUID(); const ceoId = randomUUID(); const morganId = randomUUID(); @@ -91,13 +91,13 @@ describeEmbeddedPostgres("reassigning a task whose previous assignee's run has n }); await db.insert(issues).values({ id: issueId, companyId, identifier: "REA-1", title: "Write the announcement", status: "todo", - priority: "medium", assigneeAgentId: ceoId, executionRunId: queuedRunId, + priority: "medium", assigneeAgentId: ceoId, executionRunId: options.queuedRunHoldsLock === false ? null : queuedRunId, }); return { companyId, ceoId, morganId, issueId, queuedRunId }; } - it("cancels the old assignee's never-started run and wakes the new assignee at once", async () => { - const f = await seed(); + it.each([true, false])("cancels the old assignee's never-started run and wakes the new assignee at once (run holds the lock: %s)", async queuedRunHoldsLock => { + const f = await seed({ queuedRunHoldsLock }); await request(app(f.companyId)).patch(`/api/issues/${f.issueId}`) .send({ assigneeAgentId: f.morganId }).expect(200); @@ -108,6 +108,10 @@ describeEmbeddedPostgres("reassigning a task whose previous assignee's run has n const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId)); expect(issue.assigneeAgentId).toBe(f.morganId); expect(issue.executionRunId).not.toBe(f.queuedRunId); + const newOwnerRuns = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.agentId, f.morganId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`)); + expect(newOwnerRuns).toHaveLength(1); const morganWakes = await db.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, f.companyId), eq(agentWakeupRequests.agentId, f.morganId))); expect(morganWakes).toHaveLength(1); diff --git a/server/src/__tests__/issue-update-comment-wakeup-routes.test.ts b/server/src/__tests__/issue-update-comment-wakeup-routes.test.ts index 2477493f8ba..0d11642f44f 100644 --- a/server/src/__tests__/issue-update-comment-wakeup-routes.test.ts +++ b/server/src/__tests__/issue-update-comment-wakeup-routes.test.ts @@ -47,6 +47,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async (_runId: string): Promise | null> => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async ( _runId: string, _reason: string, diff --git a/server/src/__tests__/issue-workspace-command-authz.test.ts b/server/src/__tests__/issue-workspace-command-authz.test.ts index 7772649c004..efcae950ef3 100644 --- a/server/src/__tests__/issue-workspace-command-authz.test.ts +++ b/server/src/__tests__/issue-workspace-command-authz.test.ts @@ -40,6 +40,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(), getRun: vi.fn(), getActiveRunForAgent: vi.fn(), + getQueuedAssignmentRunForIssue: vi.fn(), cancelRun: vi.fn(), })); diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index e5247fe4ed5..b78c1338523 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -1,5 +1,5 @@ import { notifyExecutionRunWork } from "../../../services/execution-control-notifications.js"; -import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js"; +import { notifyDeferredExecutionWork, notifyDeferredWakesForIssue } from "../../../services/deferred-execution-notifications.js"; import { hasRequiredWorkspaceRecovery } from "../../../services/workspace-restore-recovery-state.js"; import { readNativeWorkspaceSyncReference } from "../../../services/native-runtime/native-workspace-sync.js"; import { isPreDispatchExecutionWaitVerified } from "../../../services/pre-dispatch-execution-wait.js"; @@ -1247,6 +1247,9 @@ export function createPostgresRunDispatchAdapter( eq(issues.executionRunId, row.id), ), ); + if (decision.errorCode === "issue_assignee_changed") { + await notifyDeferredWakesForIssue(tx, { companyId: row.companyId, issueId }); + } await appendHeartbeatRunEvent(tx as unknown as Db, { companyId: row.companyId, diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index f4c944e3d5b..a2cd8346850 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -7674,14 +7674,19 @@ export function issueRoutes( } async function resolveQueuedOutgoingAssigneeRun(issue: { + id: string; companyId: string; assigneeAgentId: string | null; executionRunId?: string | null; }) { - if (!issue.executionRunId || !issue.assigneeAgentId) return null; - const run = await heartbeat.getRun(issue.executionRunId); - return run && run.status === "queued" && run.companyId === issue.companyId && - run.agentId === issue.assigneeAgentId ? run : null; + if (!issue.assigneeAgentId) return null; + const run = issue.executionRunId ? await heartbeat.getRun(issue.executionRunId) : null; + if (run && run.status === "queued" && run.companyId === issue.companyId && run.agentId === issue.assigneeAgentId) return run; + if (run && ["queued", "running", "scheduled_retry"].includes(run.status)) return null; + const assignmentRun = await heartbeat.getQueuedAssignmentRunForIssue({ + companyId: issue.companyId, agentId: issue.assigneeAgentId, issueId: issue.id, + }); + return assignmentRun ? await heartbeat.getRun(assignmentRun.id) : null; } type IssueQueueDb = Db | Parameters[0]>[0]; diff --git a/server/src/services/deferred-execution-notifications.ts b/server/src/services/deferred-execution-notifications.ts index 35878b3ec89..de9f1d1235e 100644 --- a/server/src/services/deferred-execution-notifications.ts +++ b/server/src/services/deferred-execution-notifications.ts @@ -9,6 +9,17 @@ export type DeferredReceiptRecoveryOptions = { onError?: (error: unknown) => void; }; +const ASSIGNMENT_WAKE_REASON = "issue_assigned"; + +export const assignmentWakePayloadSql = + sql`${agentWakeupRequests.payload}#>>'{_paperclipWakeContext,wakeReason}' = ${ASSIGNMENT_WAKE_REASON}`; + +export function isAssignmentWakePayload(payload: Record | null | undefined): boolean { + const context = payload?._paperclipWakeContext; + return context !== null && typeof context === "object" && !Array.isArray(context) + && (context as Record).wakeReason === ASSIGNMENT_WAKE_REASON; +} + function hint(scope: DeferredExecutionScope) { return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }); } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 32481819937..dd08e34eed0 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -903,6 +903,7 @@ export function heartbeatService( getRunIssueControl, getActiveRunIssueControlForAgent, getActiveRunForAgent, + getQueuedAssignmentRunForIssue, getActiveRunIssueSummaryForAgent, } = createHeartbeatRunState(db); const { @@ -4714,6 +4715,8 @@ export function heartbeatService( getActiveRunForAgent, + getQueuedAssignmentRunForIssue, + getActiveRunIssueSummaryForAgent, }; } diff --git a/server/src/services/heartbeat/deferred-receipt-work.ts b/server/src/services/heartbeat/deferred-receipt-work.ts index bbf49a50739..3892352a1fc 100644 --- a/server/src/services/heartbeat/deferred-receipt-work.ts +++ b/server/src/services/heartbeat/deferred-receipt-work.ts @@ -3,7 +3,7 @@ import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; import { agentWakeupRequests, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; -import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; +import { assignmentWakePayloadSql, type DeferredExecutionScope, type DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; import { dependencyResolvedWakePayloadSql } from "../issue-dependency-wakeups.js"; @@ -87,6 +87,7 @@ export function registerDeferredReceiptWork( and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`, dependencyResolvedWakePayloadSql, + assignmentWakePayloadSql, sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId} and r.runtime_mode = 'legacy' and r.status = 'cancelled' and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`, diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 8f1eecf95f1..f4d53e04744 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -13,7 +13,7 @@ import { createHeartbeatTimerPolicy } from "./timer-policy.js"; import { RECOVERY_DISPATCH_SHARE, isRecoveryWakeReason, isSystemRecoveryRun, createQueuedRunDispatcher } from "./queued-run-dispatch-order.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js"; -import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; +import { assignmentWakePayloadSql, isAssignmentWakePayload, notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; @@ -1182,6 +1182,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }); if (staleness.outcome === "cancelled" || staleness.outcome === "deferred") { applyRunDispatchPostCommitEffects(staleness.postCommitEffects); + if (staleness.outcome === "cancelled" && staleness.errorCode === "issue_assignee_changed") { + await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true, deferredPostCommitEffects }).catch((err) => { + logger.warn({ err, runId: run.id, issueId }, "claimQueuedRun: new owner's wake will be promoted by recovery"); + }); + } logger.info( { runId: run.id, @@ -2002,7 +2007,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`, sql`${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, ), sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`, - dependencyResolvedWakePayloadSql), + dependencyResolvedWakePayloadSql, assignmentWakePayloadSql), sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, sql`NOT EXISTS (SELECT 1 FROM heartbeat_runs live_issue_run WHERE live_issue_run.company_id = ${issues.companyId} @@ -2013,8 +2018,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); const releasedNativeSources = new Set(); for (const { wake } of strandedQueues) { - const dependencyWake = isDependencyResolvedWakePayload(wake.payload); - if (!dependencyWake && !queuedCommentIdsFromWakePayload(wake.payload).length && + const ownerWake = isDependencyResolvedWakePayload(wake.payload) || isAssignmentWakePayload(wake.payload); + if (!ownerWake && !queuedCommentIdsFromWakePayload(wake.payload).length && !await readQueuedInteractionResponse(db, wake.companyId, String(wake.payload?.issueId), wake.payload)) continue; const [latest] = await db.select({ id: heartbeatRuns.id, diff --git a/server/src/services/heartbeat/run-state.ts b/server/src/services/heartbeat/run-state.ts index 9a0d3c04f65..e23a84cdad5 100644 --- a/server/src/services/heartbeat/run-state.ts +++ b/server/src/services/heartbeat/run-state.ts @@ -13,6 +13,7 @@ import { eq, getTableColumns, gt, + isNull, or, sql, } from "drizzle-orm"; @@ -2057,6 +2058,18 @@ export function createHeartbeatRunState(db: Db) { return run ?? null; }, + getQueuedAssignmentRunForIssue: async (input: { companyId: string; agentId: string; issueId: string }) => { + const [run] = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.agentId, input.agentId), + eq(heartbeatRuns.status, "queued"), + isNull(heartbeatRuns.startedAt), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${input.issueId}`, + sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'issue_assigned'`, + )).orderBy(asc(heartbeatRuns.createdAt)).limit(1); + return run ?? null; + }, + getActiveRunIssueSummaryForAgent: async (agentId: string) => { const [run] = await db .select(heartbeatRunIssueSummaryColumns)