diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index e2755b0f80a..73f3bace9c2 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -202,7 +202,8 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { const blocker = vi.spyOn(executionBlocker, "getExecutionBlocker"); const admit = createPostgresIssueWakeAdmission(db); try { - await admit({ companyId, issueId }, async (tx, { fresh }) => { + await admit({ companyId, issueId }, async (tx, { fresh, lockedIssue }) => { + expect(lockedIssue).toMatchObject({ id: issueId, companyId, assigneeAgentId: agentId }); await expect(db.execute(sql`select id from issues where id = ${issueId} for update nowait`)) .rejects.toMatchObject({ cause: { code: "55P03" } }); const input = { issueId, agentId, handoffRunId, checkExecutionBlocker: true }; @@ -218,6 +219,16 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { await expect(fresh.readCommentCandidates({ ...input, receiptScope, checkExecutionBlocker: false })) .resolves.toMatchObject({ handoffAgentId: formerAgentId, candidates: [{ id: wakeId }] }); expect(blocker).toHaveBeenCalledTimes(1); + + const select = vi.spyOn(tx, "select"); + try { + expect(await executionBlocker.getExecutionBlocker(tx as unknown as Db, companyId, issueId, { + lockedIssue: lockedIssue!, + })).toMatchObject({ cause: "legacy_execution_requires_reconciliation" }); + expect(select).not.toHaveBeenCalledWith(expect.objectContaining({ conversationAgentId: issues.conversationAgentId })); + } finally { + select.mockRestore(); + } return { kind: "skipped" }; }); } finally { @@ -225,6 +236,21 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { } }); + it("locks an issue added after the initial admission lock missed", async () => { + const companyId = await seedCompany(); + const issueId = randomUUID(); + const admit = createPostgresIssueWakeAdmission(db); + + await admit({ companyId, issueId }, async (_tx, { lockedIssue, lockIssue }) => { + expect(lockedIssue).toBeNull(); + await seedIssue({ companyId, issueId }); + expect(await lockIssue()).toMatchObject({ id: issueId, companyId }); + await expect(db.execute(sql`select id from issues where id = ${issueId} for update nowait`)) + .rejects.toMatchObject({ cause: { code: "55P03" } }); + return { kind: "skipped" }; + }); + }); + function recordReleaseRunReads() { const rows: unknown[][] = []; function recordSelects>(executor: T): T { diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 9e29b287575..85bea20fecf 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1278,10 +1278,39 @@ export function createWakeAdmissionWriter(rawTx: IssueAdmissionDbTransaction, ad }; } +const issueAdmissionFields = { + id: issues.id, + companyId: issues.companyId, + identifier: issues.identifier, + conversationBoundaryCommentId: issues.conversationBoundaryCommentId, + conversationAgentId: issues.conversationAgentId, + conversationUserId: issues.conversationUserId, + conversationState: issues.conversationState, + status: issues.status, + statusVersion: issues.statusVersion, + projectId: issues.projectId, + projectWorkspaceId: issues.projectWorkspaceId, + executionWorkspaceId: issues.executionWorkspaceId, + executionWorkspacePreference: issues.executionWorkspacePreference, + executionWorkspaceSettings: issues.executionWorkspaceSettings, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + monitorNextCheckAt: issues.monitorNextCheckAt, + monitorWakeRequestedAt: issues.monitorWakeRequestedAt, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + executionAgentNameKey: issues.executionAgentNameKey, + createdAt: issues.createdAt, +}; + +export type IssueAdmissionRow = Pick; + export function createPostgresIssueWakeAdmission(db: Db) { return async function withIssueAdmission( { companyId, issueId, durableChatRequest }: IssueAdmissionTarget, evaluate: (tx: IssueAdmissionDbTransaction, stores: { + lockedIssue: IssueAdmissionRow | null; + lockIssue: () => Promise; reader: WakeAdmissionReader; writer: WakeAdmissionWriter; fresh: FreshIssueRunStore; @@ -1290,8 +1319,15 @@ export function createPostgresIssueWakeAdmission(db: Db) { try { return await db.transaction(withDurableChatAdmission(durableChatRequest, async (tx) => { // Mentions may target another agent, but every admission must share the issue lock. - await tx.execute(sql`select id from issues where id = ${issueId} and company_id = ${companyId} for update`); + const lockIssue = async () => { + const [issue] = await tx.select(issueAdmissionFields).from(issues) + .where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))).for("update"); + return issue ?? null; + }; + const lockedIssue = await lockIssue(); return evaluate(tx, { + lockedIssue, + lockIssue, reader: createWakeAdmissionReader(tx, companyId), writer: createWakeAdmissionWriter(tx, companyId), fresh: createFreshIssueRunStore(tx, companyId), diff --git a/server/src/modules/wake-queue/index.ts b/server/src/modules/wake-queue/index.ts index e6c0819d013..96dec6b04e8 100644 --- a/server/src/modules/wake-queue/index.ts +++ b/server/src/modules/wake-queue/index.ts @@ -3,6 +3,7 @@ import { createPostgresIssueWakeAdmission, createPostgresWakeQueueAdapter, type IssueAdmissionDbTransaction, + type IssueAdmissionRow, type IssueAdmissionTarget, } from "./adapters/postgres.js"; import { createQueuedCommentIssueLockWriter } from "./adapters/queued-comment-postgres.js"; @@ -82,6 +83,8 @@ export type WakeQueueDeps = { type HeartbeatRun = typeof heartbeatRuns.$inferSelect; type LockedIssueAdmission = { + lockedIssue: IssueAdmissionRow | null; + lockIssue: () => Promise; admitWakeBehindIssueExecution: ReturnType; admitFreshIssueRun: ReturnType>; }; @@ -103,10 +106,12 @@ export function createWakeQueue(db: Db, deps: WakeQueueDeps) { evaluateHeartbeatPolicy: (tx: IssueAdmissionDbTransaction, admission: LockedIssueAdmission) => Promise>, effects: IssueWakeAdmissionEffects, ) { - return admitIssueWake(() => withIssueAdmission({ companyId, issueId, durableChatRequest }, (tx, { reader, writer, fresh }) => { + return admitIssueWake(() => withIssueAdmission({ companyId, issueId, durableChatRequest }, (tx, { lockedIssue, lockIssue, reader, writer, fresh }) => { const admitBusy = createAdmitWakeBehindIssueExecution({ reader, writer, helpers: deps.wakeAdmissionHelpers }); const admitFresh = createFreshIssueRunAdmission(fresh); return evaluateHeartbeatPolicy(tx, { + lockedIssue, + lockIssue, async admitWakeBehindIssueExecution(input) { if (input.companyId !== companyId || input.issueId !== issueId || input.contextSnapshot.issueId !== issueId) { throw new Error("wake-queue: admission does not match the locked company and issue"); diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index a5e84a855ec..ed95748d982 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -321,17 +321,26 @@ export function restoreHoldPredicate() { ); } +const blockerIssueFields = { + id: issues.id, companyId: issues.companyId, + conversationAgentId: issues.conversationAgentId, conversationBoundaryCommentId: issues.conversationBoundaryCommentId, + status: issues.status, assigneeAgentId: issues.assigneeAgentId, statusVersion: issues.statusVersion, + executionRunId: issues.executionRunId, checkoutRunId: issues.checkoutRunId, +}; + export async function getExecutionBlocker(db: Db, companyId: string, issueId: string, options?: { conversationResetCommentId?: string | null; + /** The current issue row held under this transaction's issue lock. */ + lockedIssue?: Pick; /** The native run being dispatched; its own scheduled resume does not block it. */ resumingNativeRunId?: string | null; }): Promise { - const [conversation] = await db.select({ agentId: issues.conversationAgentId, - boundaryId: issues.conversationBoundaryCommentId, status: issues.status, - assigneeAgentId: issues.assigneeAgentId, statusVersion: issues.statusVersion, - executionRunId: issues.executionRunId, checkoutRunId: issues.checkoutRunId }).from(issues).where(and( - eq(issues.companyId, companyId), eq(issues.id, issueId), - )).limit(1); + const lockedIssue = options?.lockedIssue; + const conversation = lockedIssue?.id === issueId && lockedIssue.companyId === companyId + ? lockedIssue + : (await db.select(blockerIssueFields).from(issues).where(and( + eq(issues.companyId, companyId), eq(issues.id, issueId), + )).limit(1))[0]; // Resetting model context cannot repair or recover a workspace. This hold // survives conversation boundaries until the existing repair/reconciliation path clears it. const [restoreHold] = await db.select().from(issueRecoveryActions).where(and( @@ -343,17 +352,17 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st )).orderBy(desc(issueRecoveryActions.updatedAt)).limit(1); // A persisted user /new is an ordered context command, not a retry of uncertain work. // The normal issue execution lock still serializes it behind any active turn. - if (!restoreHold && conversation?.agentId && options?.conversationResetCommentId) { + if (!restoreHold && conversation?.conversationAgentId && options?.conversationResetCommentId) { const [command] = await db.select().from(issueComments).where(and( eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, options.conversationResetCommentId), )).limit(1); if (command?.authorUserId && !command.deletedAt && command.body.trim() === "/new") return null; } - const [boundary] = conversation?.agentId && conversation.boundaryId + const [boundary] = conversation?.conversationAgentId && conversation.conversationBoundaryCommentId ? await db.select({ createdAt: issueComments.createdAt }).from(issueComments).where(and( eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), - eq(issueComments.id, conversation.boundaryId), + eq(issueComments.id, conversation.conversationBoundaryCommentId), )).limit(1) : []; const ownership = await getConversationOwnershipBlocker(db, companyId, issueId); diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index 9a7e8717144..3f0d8757902 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -288,6 +288,7 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { const admit = deps.wakeQueue.admitIssueWake.getMockImplementation()!; deps.wakeQueue.admitIssueWake.mockImplementation((target, evaluate, effects) => admit(target, (tx, admission) => evaluate(tx, otherIssue ? { + ...admission, admitFreshIssueRun: (input) => admission.admitFreshIssueRun({ ...input, issue: { ...input.issue, id: otherIssue.id }, contextSnapshot: { ...input.contextSnapshot, issueId: otherIssue.id } }), admitWakeBehindIssueExecution: (input) => admission.admitWakeBehindIssueExecution({ ...input, @@ -309,6 +310,34 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { expect(await db.select().from(heartbeatRuns)).toHaveLength(kind === "fresh" ? 0 : 1); }); + it("uses issue changes written by durable chat authorization before admission", async () => { + const { user, company, agent, issue, queue } = await waitingConversationFixture(); + await db.update(issues).set({ conversationAgentId: null, conversationUserId: null, conversationState: null, status: "todo" }) + .where(eq(issues.id, issue.id)); + const [comment] = await db.insert(issueComments).values({ companyId: company.id, issueId: issue.id, + authorType: "user", authorUserId: user.id, body: "Please continue" }).returning(); + const requestId = randomUUID(); + const request = createDurableChatWakeupRequest({ + id: requestId, companyId: company.id, issueId: issue.id, agentId: agent.id, commentId: comment.id, + requestedByActorType: "user", requestedByActorId: user.id, requestedAt: new Date(), + authorize: async (tx) => { + await tx.update(issues).set({ status: "blocked" }).where(eq(issues.id, issue.id)); + }, + }); + + await expect(queue.enqueueWakeup(agent.id, { + source: "assignment", triggerDetail: "system", reason: "External chat message received", + requestedByActorType: "user", requestedByActorId: user.id, durableChatRequest: request, + payload: { issueId: issue.id, commentId: comment.id, mutation: "chat_message_received" }, + issueStateGuard: { assigneeAgentId: agent.id, statuses: ["todo"] }, + })).resolves.toBeNull(); + + const [receipt] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, requestId)); + expect(receipt).toMatchObject({ status: "skipped", reason: "issue_state_guard_mismatch", + payload: { heartbeatSkip: { actualStatus: "blocked" } } }); + expect(await db.select().from(heartbeatRuns)).toEqual([]); + }); + it("admits the owner's context compaction maintenance request while its conversation waits", async () => { const f = await waitingCompactionFixture(); const accepted = await f.service.request({ companyId: f.company.id, issue: f.issue, userId: f.user.id }); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index a3a7cb484d5..8a74404ace2 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -3528,7 +3528,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende return wakeQueue.admitIssueWake( { companyId: agent.companyId, issueId, durableChatRequest: durableRequest }, - async (tx, { admitWakeBehindIssueExecution, admitFreshIssueRun }) => { + async (tx, { lockedIssue, lockIssue, admitWakeBehindIssueExecution, admitFreshIssueRun }) => { if (executionWaitRequestId) { const [pending] = await tx.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.id, executionWaitRequestId), eq(agentWakeupRequests.companyId, agent.companyId), @@ -3781,37 +3781,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } } - const issue = await tx - .select({ - id: issues.id, - companyId: issues.companyId, - identifier: issues.identifier, - conversationAgentId: issues.conversationAgentId, - conversationUserId: issues.conversationUserId, - conversationState: issues.conversationState, - status: issues.status, - statusVersion: issues.statusVersion, - projectId: issues.projectId, - projectWorkspaceId: issues.projectWorkspaceId, - executionWorkspaceId: issues.executionWorkspaceId, - executionWorkspacePreference: issues.executionWorkspacePreference, - executionWorkspaceSettings: issues.executionWorkspaceSettings, - assigneeAgentId: issues.assigneeAgentId, - assigneeUserId: issues.assigneeUserId, - monitorNextCheckAt: issues.monitorNextCheckAt, - monitorWakeRequestedAt: issues.monitorWakeRequestedAt, - executionRunId: issues.executionRunId, - executionAgentNameKey: issues.executionAgentNameKey, - createdAt: issues.createdAt, - }) - .from(issues) - .where( - and( - eq(issues.id, issueId), - eq(issues.companyId, agent.companyId), - ), - ) - .then((rows) => rows[0] ?? null); + // Durable chat authorization can write through this transaction. A missing + // row has no lock, so the later read can also see a newly inserted issue. + const issue = durableRequest || !lockedIssue ? await lockIssue() : lockedIssue; if (!issue) { if (caseRequest) throw conflict("Case handoff task no longer exists"); @@ -4110,11 +4082,16 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende )).limit(1) : []; const verifiedConversationReset = Boolean(resetComment && isConversationReset(resetComment.body)); - if (wakeCommentId && opts.requestedByActorType === "user") + const preserveWorkspace = wakeCommentId && opts.requestedByActorType === "user" + && issue.executionWorkspaceId !== null; + if (preserveWorkspace) await preserveTaskRemoteWorkspaceWriter(tx as unknown as Db, issue.companyId, issue.id); const executionBlocker = await getExecutionBlocker( tx as unknown as Db, issue.companyId, issue.id, - { conversationResetCommentId: verifiedConversationReset ? wakeCommentId : null }, + { + conversationResetCommentId: verifiedConversationReset ? wakeCommentId : null, + lockedIssue: preserveWorkspace ? undefined : issue, + }, ); // Prove eligibility without retiring the hold. Later gates can still // decline this wake; hold retirement and successor creation stay atomic.