diff --git a/server/src/__tests__/artifact-review-document-routes.test.ts b/server/src/__tests__/artifact-review-document-routes.test.ts index 2a34aa2efb0..d926a954405 100644 --- a/server/src/__tests__/artifact-review-document-routes.test.ts +++ b/server/src/__tests__/artifact-review-document-routes.test.ts @@ -81,6 +81,7 @@ function registerRouteMocks() { reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), }), instanceSettingsService: () => ({ diff --git a/server/src/__tests__/board-native-attachments.integration.test.ts b/server/src/__tests__/board-native-attachments.integration.test.ts index b5e5f493720..1374e6261c9 100644 --- a/server/src/__tests__/board-native-attachments.integration.test.ts +++ b/server/src/__tests__/board-native-attachments.integration.test.ts @@ -51,6 +51,7 @@ vi.mock("../services/heartbeat.js", async (importOriginal) => ({ wakeup, getRun: async () => null, getActiveRunForAgent: async () => null, + getQueuedAssignmentRunForIssue: async () => null, reportRunActivity: async () => undefined, }), })); diff --git a/server/src/__tests__/environment-selection-route-guards.test.ts b/server/src/__tests__/environment-selection-route-guards.test.ts index c08a109f8c0..f3e5a34e820 100644 --- a/server/src/__tests__/environment-selection-route-guards.test.ts +++ b/server/src/__tests__/environment-selection-route-guards.test.ts @@ -84,6 +84,7 @@ vi.mock("../services/index.js", () => ({ heartbeatService: () => ({ getRun: vi.fn(), getActiveRunForAgent: vi.fn(), + getQueuedAssignmentRunForIssue: vi.fn(), }), issueApprovalService: () => ({ listApprovalsForIssue: vi.fn(), diff --git a/server/src/__tests__/external-object-routes.test.ts b/server/src/__tests__/external-object-routes.test.ts index 7df975fe682..738b5947b10 100644 --- a/server/src/__tests__/external-object-routes.test.ts +++ b/server/src/__tests__/external-object-routes.test.ts @@ -71,6 +71,7 @@ function registerRouteMocks() { reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), }), issueApprovalService: () => ({}), diff --git a/server/src/__tests__/heartbeat-reassignment-deferred-wake.test.ts b/server/src/__tests__/heartbeat-reassignment-deferred-wake.test.ts new file mode 100644 index 00000000000..f5556975be1 --- /dev/null +++ b/server/src/__tests__/heartbeat-reassignment-deferred-wake.test.ts @@ -0,0 +1,151 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, + agentWakeupRequests, + authUsers, + companies, + companyMemberships, + createDb, + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, + heartbeatRuns, + issues, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js"; +import { truncateTablesWithDeadlockRetry } from "./helpers/truncate-with-deadlock-retry.js"; +import { heartbeatService } from "../services/heartbeat.ts"; +import { runningProcesses, type AdapterExecutionContext, type AdapterExecutionResult } from "../adapters/index.ts"; + +const mockAdapterExecute = vi.hoisted(() => + vi.fn(async (_context: AdapterExecutionContext): Promise => ({ + exitCode: 0, signal: null, timedOut: false, errorMessage: null, + summary: "Reassignment wake test run.", provider: "test", model: "test-model", + })), +); + +vi.mock("../adapters/index.ts", async () => { + const actual = await vi.importActual("../adapters/index.ts"); + return { ...actual, getServerAdapter: vi.fn(() => ({ supportsLocalAgentJwt: false, execute: mockAdapterExecute })) }; +}); + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping reassignment deferred-wake tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +describeEmbeddedPostgres("the new owner's wake after a task is reassigned away from a queued creator run", () => { + let db!: ReturnType; + let heartbeat!: ReturnType; + let tempDb: Awaited> | null = null; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-reassign-deferred-wake-"); + db = createDb(tempDb.connectionString); + heartbeat = heartbeatService(db, { runtimeEnv: { ...process.env, PAPERCLIP_IN_WORKTREE: "false" } }); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + afterEach(async () => { + mockAdapterExecute.mockClear(); + runningProcesses.clear(); + await drainHeartbeatRunsToQuiescence(db, heartbeat); + await truncateTablesWithDeadlockRetry(db, `TRUNCATE TABLE "issues", "heartbeat_run_events", "activity_log", + "heartbeat_runs", "agent_wakeup_requests", "agent_runtime_state", "agents", "companies" RESTART IDENTITY CASCADE`, + { attempts: 10, delayMs: () => 100 }); + await db.delete(authUsers).where(eq(authUsers.id, "responsible-user")); + }); + + afterAll(async () => { + await db?.$client.end(); + await tempDb?.cleanup(); + }); + + async function seedReassignedTaskWithDeferredNewOwner() { + const companyId = randomUUID(); + const ceoId = randomUUID(); + const morganId = randomUUID(); + const issueId = randomUUID(); + const ceoRunId = randomUUID(); + const ceoWakeId = randomUUID(); + await db.insert(companies).values({ + id: companyId, name: "Reassign Co", issuePrefix: `R${companyId.replace(/-/g, "").slice(0, 5).toUpperCase()}`, + defaultResponsibleUserId: "responsible-user", requireBoardApprovalForNewAgents: false, + }); + await db.insert(authUsers).values({ + id: "responsible-user", name: "Fixture user", email: "responsible-user@example.test", + createdAt: new Date(), updatedAt: new Date(), + }).onConflictDoNothing(); + await db.insert(companyMemberships).values({ + companyId, principalType: "user", principalId: "responsible-user", status: "active", membershipRole: "member", + }); + await db.insert(agents).values([ceoId, morganId].map((id, index) => ({ + id, companyId, name: index === 0 ? "CEO" : "Morgan", role: index === 0 ? "ceo" : "engineer", + status: "idle", adapterType: "codex_local", adapterConfig: {}, + runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } }, permissions: {}, + }))); + await db.insert(issues).values({ + id: issueId, companyId, title: "Make a simple website", status: "todo", priority: "medium", + assigneeAgentId: morganId, + }); + await db.insert(agentWakeupRequests).values({ + id: ceoWakeId, companyId, agentId: ceoId, source: "assignment", triggerDetail: "system", + reason: "issue_assigned", payload: { issueId }, status: "queued", + }); + await db.insert(heartbeatRuns).values({ + id: ceoRunId, companyId, agentId: ceoId, invocationSource: "assignment", triggerDetail: "system", + status: "queued", wakeupRequestId: ceoWakeId, contextSnapshot: { issueId, wakeReason: "issue_assigned" }, + }); + await db.update(agentWakeupRequests).set({ runId: ceoRunId }).where(eq(agentWakeupRequests.id, ceoWakeId)); + + await heartbeat.wakeup(morganId, { + source: "assignment", triggerDetail: "system", reason: "issue_assigned", + requestedByActorType: "system", payload: { issueId }, + }); + const [deferred] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, morganId)); + expect(deferred).toMatchObject({ status: "deferred_issue_execution", runId: null }); + return { companyId, ceoId, morganId, issueId, ceoRunId, deferredWakeId: deferred!.id }; + } + + async function expectOneNewOwnerRun(f: { morganId: string; issueId: string; deferredWakeId: string }) { + await vi.waitFor(async () => { + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.deferredWakeId)); + expect(wake!.status).not.toBe("deferred_issue_execution"); + expect(wake!.runId).toBeTruthy(); + const runs = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.agentId, f.morganId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`)); + expect(runs).toHaveLength(1); + expect(runs[0]!.id).toBe(wake!.runId); + expect(mockAdapterExecute.mock.calls.filter(([context]) => context?.runId === wake!.runId), JSON.stringify({ status: runs[0]!.status, errorCode: runs[0]!.errorCode, error: runs[0]!.error })).toHaveLength(1); + }, { timeout: 15_000 }); + } + + it("promotes the new owner's wake when the queue withholds the stale creator run", async () => { + const f = await seedReassignedTaskWithDeferredNewOwner(); + + await heartbeat.resumeQueuedRuns(); + + const [stale] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.ceoRunId)); + expect(stale).toMatchObject({ status: "cancelled", errorCode: "issue_assignee_changed" }); + expect(mockAdapterExecute.mock.calls.filter(([context]) => context?.runId === f.ceoRunId)).toHaveLength(0); + await expectOneNewOwnerRun(f); + }); + + it("recovers a new owner's wake left deferred after the blocking run already finished", async () => { + const f = await seedReassignedTaskWithDeferredNewOwner(); + await db.update(heartbeatRuns).set({ + status: "cancelled", finishedAt: new Date(), errorCode: "issue_assignee_changed", + error: "Cancelled because issue assignee changed before the queued run could start; the new owner will be woken instead", + }).where(eq(heartbeatRuns.id, f.ceoRunId)); + await db.update(issues).set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null }) + .where(eq(issues.id, f.issueId)); + + await heartbeat.resumeQueuedRuns(); + + await expectOneNewOwnerRun(f); + }); +}); diff --git a/server/src/__tests__/issue-activity-events-routes.test.ts b/server/src/__tests__/issue-activity-events-routes.test.ts index 1fc4088c911..14a4cb6e93d 100644 --- a/server/src/__tests__/issue-activity-events-routes.test.ts +++ b/server/src/__tests__/issue-activity-events-routes.test.ts @@ -36,6 +36,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), })); const mockFeedbackService = vi.hoisted(() => ({ diff --git a/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts b/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts index d31598d2555..b7f16117cab 100644 --- a/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts +++ b/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts @@ -125,6 +125,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async (): Promise => null), getActiveRunForAgent: vi.fn(async (): Promise => null), + getQueuedAssignmentRunForIssue: vi.fn(async (): Promise => null), cancelRun: vi.fn(async (): Promise => null), resumeTaskRetry: vi.fn(async () => undefined), })); diff --git a/server/src/__tests__/issue-assignee-invokability-routes.test.ts b/server/src/__tests__/issue-assignee-invokability-routes.test.ts index 45f0b2e88c6..d6a788b8f13 100644 --- a/server/src/__tests__/issue-assignee-invokability-routes.test.ts +++ b/server/src/__tests__/issue-assignee-invokability-routes.test.ts @@ -38,6 +38,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), })); diff --git a/server/src/__tests__/issue-attachment-routes.test.ts b/server/src/__tests__/issue-attachment-routes.test.ts index ed9451ad0c9..6afe4b14f2a 100644 --- a/server/src/__tests__/issue-attachment-routes.test.ts +++ b/server/src/__tests__/issue-attachment-routes.test.ts @@ -67,6 +67,7 @@ function registerRouteMocks() { reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), }), instanceSettingsService: () => ({ diff --git a/server/src/__tests__/issue-closed-workspace-routes.test.ts b/server/src/__tests__/issue-closed-workspace-routes.test.ts index 9d109d1a63f..ca78112128c 100644 --- a/server/src/__tests__/issue-closed-workspace-routes.test.ts +++ b/server/src/__tests__/issue-closed-workspace-routes.test.ts @@ -44,6 +44,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), })); diff --git a/server/src/__tests__/issue-comment-cancel-routes.test.ts b/server/src/__tests__/issue-comment-cancel-routes.test.ts index 410f697e7b5..88fb71596f0 100644 --- a/server/src/__tests__/issue-comment-cancel-routes.test.ts +++ b/server/src/__tests__/issue-comment-cancel-routes.test.ts @@ -23,6 +23,7 @@ const mockAccessService = vi.hoisted(() => ({ const mockHeartbeatService = vi.hoisted(() => ({ getRun: vi.fn(async (): Promise => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), })); const mockAuthoritativeQueueWakes = vi.hoisted(() => [] as Array>); diff --git a/server/src/__tests__/issue-comment-reopen-routes.test.ts b/server/src/__tests__/issue-comment-reopen-routes.test.ts index 1ade5a31de9..1a1f7d52f1b 100644 --- a/server/src/__tests__/issue-comment-reopen-routes.test.ts +++ b/server/src/__tests__/issue-comment-reopen-routes.test.ts @@ -40,6 +40,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async (): Promise => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async (): Promise => null), })); diff --git a/server/src/__tests__/issue-execution-policy-routes.test.ts b/server/src/__tests__/issue-execution-policy-routes.test.ts index 8fffd3761f1..bda24c1dfee 100644 --- a/server/src/__tests__/issue-execution-policy-routes.test.ts +++ b/server/src/__tests__/issue-execution-policy-routes.test.ts @@ -25,6 +25,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), })); diff --git a/server/src/__tests__/issue-feedback-routes.test.ts b/server/src/__tests__/issue-feedback-routes.test.ts index 643956ae223..13ae62a9da5 100644 --- a/server/src/__tests__/issue-feedback-routes.test.ts +++ b/server/src/__tests__/issue-feedback-routes.test.ts @@ -31,6 +31,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async () => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async () => null), })); const mockInstanceSettingsService = vi.hoisted(() => ({ diff --git a/server/src/__tests__/issue-reassignment-queued-run.test.ts b/server/src/__tests__/issue-reassignment-queued-run.test.ts index 151e9fc5799..4a3b3e4bf20 100644 --- a/server/src/__tests__/issue-reassignment-queued-run.test.ts +++ b/server/src/__tests__/issue-reassignment-queued-run.test.ts @@ -65,7 +65,7 @@ describeEmbeddedPostgres("reassigning a task whose previous assignee's run has n return testApp; } - async function seed() { + async function seed(options: { queuedRunHoldsLock?: boolean } = {}) { const companyId = randomUUID(); const ceoId = randomUUID(); const morganId = randomUUID(); @@ -91,13 +91,13 @@ describeEmbeddedPostgres("reassigning a task whose previous assignee's run has n }); await db.insert(issues).values({ id: issueId, companyId, identifier: "REA-1", title: "Write the announcement", status: "todo", - priority: "medium", assigneeAgentId: ceoId, executionRunId: queuedRunId, + priority: "medium", assigneeAgentId: ceoId, executionRunId: options.queuedRunHoldsLock === false ? null : queuedRunId, }); return { companyId, ceoId, morganId, issueId, queuedRunId }; } - it("cancels the old assignee's never-started run and wakes the new assignee at once", async () => { - const f = await seed(); + it.each([true, false])("cancels the old assignee's never-started run and wakes the new assignee at once (run holds the lock: %s)", async queuedRunHoldsLock => { + const f = await seed({ queuedRunHoldsLock }); await request(app(f.companyId)).patch(`/api/issues/${f.issueId}`) .send({ assigneeAgentId: f.morganId }).expect(200); @@ -108,6 +108,10 @@ describeEmbeddedPostgres("reassigning a task whose previous assignee's run has n const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId)); expect(issue.assigneeAgentId).toBe(f.morganId); expect(issue.executionRunId).not.toBe(f.queuedRunId); + const newOwnerRuns = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.agentId, f.morganId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`)); + expect(newOwnerRuns).toHaveLength(1); const morganWakes = await db.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, f.companyId), eq(agentWakeupRequests.agentId, f.morganId))); expect(morganWakes).toHaveLength(1); diff --git a/server/src/__tests__/issue-update-comment-wakeup-routes.test.ts b/server/src/__tests__/issue-update-comment-wakeup-routes.test.ts index 2477493f8ba..0d11642f44f 100644 --- a/server/src/__tests__/issue-update-comment-wakeup-routes.test.ts +++ b/server/src/__tests__/issue-update-comment-wakeup-routes.test.ts @@ -47,6 +47,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(async () => undefined), getRun: vi.fn(async (_runId: string): Promise | null> => null), getActiveRunForAgent: vi.fn(async () => null), + getQueuedAssignmentRunForIssue: vi.fn(async () => null), cancelRun: vi.fn(async ( _runId: string, _reason: string, diff --git a/server/src/__tests__/issue-workspace-command-authz.test.ts b/server/src/__tests__/issue-workspace-command-authz.test.ts index 7772649c004..efcae950ef3 100644 --- a/server/src/__tests__/issue-workspace-command-authz.test.ts +++ b/server/src/__tests__/issue-workspace-command-authz.test.ts @@ -40,6 +40,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ reportRunActivity: vi.fn(), getRun: vi.fn(), getActiveRunForAgent: vi.fn(), + getQueuedAssignmentRunForIssue: vi.fn(), cancelRun: vi.fn(), })); diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index e5247fe4ed5..b78c1338523 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -1,5 +1,5 @@ import { notifyExecutionRunWork } from "../../../services/execution-control-notifications.js"; -import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js"; +import { notifyDeferredExecutionWork, notifyDeferredWakesForIssue } from "../../../services/deferred-execution-notifications.js"; import { hasRequiredWorkspaceRecovery } from "../../../services/workspace-restore-recovery-state.js"; import { readNativeWorkspaceSyncReference } from "../../../services/native-runtime/native-workspace-sync.js"; import { isPreDispatchExecutionWaitVerified } from "../../../services/pre-dispatch-execution-wait.js"; @@ -1247,6 +1247,9 @@ export function createPostgresRunDispatchAdapter( eq(issues.executionRunId, row.id), ), ); + if (decision.errorCode === "issue_assignee_changed") { + await notifyDeferredWakesForIssue(tx, { companyId: row.companyId, issueId }); + } await appendHeartbeatRunEvent(tx as unknown as Db, { companyId: row.companyId, diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index f4c944e3d5b..a2cd8346850 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -7674,14 +7674,19 @@ export function issueRoutes( } async function resolveQueuedOutgoingAssigneeRun(issue: { + id: string; companyId: string; assigneeAgentId: string | null; executionRunId?: string | null; }) { - if (!issue.executionRunId || !issue.assigneeAgentId) return null; - const run = await heartbeat.getRun(issue.executionRunId); - return run && run.status === "queued" && run.companyId === issue.companyId && - run.agentId === issue.assigneeAgentId ? run : null; + if (!issue.assigneeAgentId) return null; + const run = issue.executionRunId ? await heartbeat.getRun(issue.executionRunId) : null; + if (run && run.status === "queued" && run.companyId === issue.companyId && run.agentId === issue.assigneeAgentId) return run; + if (run && ["queued", "running", "scheduled_retry"].includes(run.status)) return null; + const assignmentRun = await heartbeat.getQueuedAssignmentRunForIssue({ + companyId: issue.companyId, agentId: issue.assigneeAgentId, issueId: issue.id, + }); + return assignmentRun ? await heartbeat.getRun(assignmentRun.id) : null; } type IssueQueueDb = Db | Parameters[0]>[0]; diff --git a/server/src/services/deferred-execution-notifications.ts b/server/src/services/deferred-execution-notifications.ts index 35878b3ec89..de9f1d1235e 100644 --- a/server/src/services/deferred-execution-notifications.ts +++ b/server/src/services/deferred-execution-notifications.ts @@ -9,6 +9,17 @@ export type DeferredReceiptRecoveryOptions = { onError?: (error: unknown) => void; }; +const ASSIGNMENT_WAKE_REASON = "issue_assigned"; + +export const assignmentWakePayloadSql = + sql`${agentWakeupRequests.payload}#>>'{_paperclipWakeContext,wakeReason}' = ${ASSIGNMENT_WAKE_REASON}`; + +export function isAssignmentWakePayload(payload: Record | null | undefined): boolean { + const context = payload?._paperclipWakeContext; + return context !== null && typeof context === "object" && !Array.isArray(context) + && (context as Record).wakeReason === ASSIGNMENT_WAKE_REASON; +} + function hint(scope: DeferredExecutionScope) { return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }); } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 32481819937..dd08e34eed0 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -903,6 +903,7 @@ export function heartbeatService( getRunIssueControl, getActiveRunIssueControlForAgent, getActiveRunForAgent, + getQueuedAssignmentRunForIssue, getActiveRunIssueSummaryForAgent, } = createHeartbeatRunState(db); const { @@ -4714,6 +4715,8 @@ export function heartbeatService( getActiveRunForAgent, + getQueuedAssignmentRunForIssue, + getActiveRunIssueSummaryForAgent, }; } diff --git a/server/src/services/heartbeat/deferred-receipt-work.ts b/server/src/services/heartbeat/deferred-receipt-work.ts index bbf49a50739..3892352a1fc 100644 --- a/server/src/services/heartbeat/deferred-receipt-work.ts +++ b/server/src/services/heartbeat/deferred-receipt-work.ts @@ -3,7 +3,7 @@ import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; import { agentWakeupRequests, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; -import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; +import { assignmentWakePayloadSql, type DeferredExecutionScope, type DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; import { dependencyResolvedWakePayloadSql } from "../issue-dependency-wakeups.js"; @@ -87,6 +87,7 @@ export function registerDeferredReceiptWork( and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`, dependencyResolvedWakePayloadSql, + assignmentWakePayloadSql, sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId} and r.runtime_mode = 'legacy' and r.status = 'cancelled' and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`, diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 8f1eecf95f1..f4d53e04744 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -13,7 +13,7 @@ import { createHeartbeatTimerPolicy } from "./timer-policy.js"; import { RECOVERY_DISPATCH_SHARE, isRecoveryWakeReason, isSystemRecoveryRun, createQueuedRunDispatcher } from "./queued-run-dispatch-order.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js"; -import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; +import { assignmentWakePayloadSql, isAssignmentWakePayload, notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; @@ -1182,6 +1182,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }); if (staleness.outcome === "cancelled" || staleness.outcome === "deferred") { applyRunDispatchPostCommitEffects(staleness.postCommitEffects); + if (staleness.outcome === "cancelled" && staleness.errorCode === "issue_assignee_changed") { + await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true, deferredPostCommitEffects }).catch((err) => { + logger.warn({ err, runId: run.id, issueId }, "claimQueuedRun: new owner's wake will be promoted by recovery"); + }); + } logger.info( { runId: run.id, @@ -2002,7 +2007,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`, sql`${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, ), sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`, - dependencyResolvedWakePayloadSql), + dependencyResolvedWakePayloadSql, assignmentWakePayloadSql), sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, sql`NOT EXISTS (SELECT 1 FROM heartbeat_runs live_issue_run WHERE live_issue_run.company_id = ${issues.companyId} @@ -2013,8 +2018,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); const releasedNativeSources = new Set(); for (const { wake } of strandedQueues) { - const dependencyWake = isDependencyResolvedWakePayload(wake.payload); - if (!dependencyWake && !queuedCommentIdsFromWakePayload(wake.payload).length && + const ownerWake = isDependencyResolvedWakePayload(wake.payload) || isAssignmentWakePayload(wake.payload); + if (!ownerWake && !queuedCommentIdsFromWakePayload(wake.payload).length && !await readQueuedInteractionResponse(db, wake.companyId, String(wake.payload?.issueId), wake.payload)) continue; const [latest] = await db.select({ id: heartbeatRuns.id, diff --git a/server/src/services/heartbeat/run-state.ts b/server/src/services/heartbeat/run-state.ts index 9a0d3c04f65..e23a84cdad5 100644 --- a/server/src/services/heartbeat/run-state.ts +++ b/server/src/services/heartbeat/run-state.ts @@ -13,6 +13,7 @@ import { eq, getTableColumns, gt, + isNull, or, sql, } from "drizzle-orm"; @@ -2057,6 +2058,18 @@ export function createHeartbeatRunState(db: Db) { return run ?? null; }, + getQueuedAssignmentRunForIssue: async (input: { companyId: string; agentId: string; issueId: string }) => { + const [run] = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.agentId, input.agentId), + eq(heartbeatRuns.status, "queued"), + isNull(heartbeatRuns.startedAt), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${input.issueId}`, + sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'issue_assigned'`, + )).orderBy(asc(heartbeatRuns.createdAt)).limit(1); + return run ?? null; + }, + getActiveRunIssueSummaryForAgent: async (agentId: string) => { const [run] = await db .select(heartbeatRunIssueSummaryColumns)