diff --git a/doc/heartbeat-internals.md b/doc/heartbeat-internals.md index ee6b4b7b282..f89d45b0992 100644 --- a/doc/heartbeat-internals.md +++ b/doc/heartbeat-internals.md @@ -250,8 +250,11 @@ wake and run rows, so edits, ordering, interruption, and saved execution inputs keep their existing write contract. The task composer offers Steer when the native session has an active provider -turn that accepts steering. If steering is unavailable or unsupported, it offers -an explicit Interrupt action instead. Interrupt records the board user's intent, +turn that accepts steering. Wake-queue locks the task and receipt for the click, +then locks and rereads the run after the provider acknowledges delivery. It +records the acknowledgement and removes the queued entry in one transaction. +If steering is unavailable or unsupported, the composer offers Interrupt. +Interrupt records the board user's intent, requests cancellation, and delivers the saved queue through normal admission after the previous execution stops. Preparation and cleanup can keep messages queued; the action does not bypass Stop confirmation or execution gates. diff --git a/server/src/__tests__/issue-queued-comments-routes.test.ts b/server/src/__tests__/issue-queued-comments-routes.test.ts index dd1f9b10719..3a2433a4bf5 100644 --- a/server/src/__tests__/issue-queued-comments-routes.test.ts +++ b/server/src/__tests__/issue-queued-comments-routes.test.ts @@ -357,6 +357,9 @@ describeEmbeddedPostgres("issue queued-comment routes", () => { await request(client).post(`/api/issues/${seeded.issueId}/queued-comments/interrupt`).send({ queueId: seeded.wakeId, revision: queue.body.revision, targetRunId: null, }).expect(200); + const interruption = await db.select({ details: activityLog.details }).from(activityLog) + .where(eq(activityLog.action, "issue.queued_comments_interrupted")); + expect(interruption).toEqual([{ details: { queueId: seeded.wakeId, targetRunId: null } }]); expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, seeded.wakeId)))[0].status) .toBe("deferred_issue_execution"); // Cleanup timestamps alone are not proof. A mismatched receipt is not proof either. @@ -2261,11 +2264,14 @@ describeEmbeddedPostgres("issue queued-comment routes", () => { expect(steered.body.entries.map((entry: any) => entry.comment.id)).toEqual([seeded.commentIds[1]]); }); - it.each(["cancelled", "failed", "succeeded"] as const)("preserves queued input when the run stops during acknowledgement (%s)", async status => { + it.each(["cancelled", "failed", "succeeded", "different task"] as const)("preserves queued input when the run stops or changes scope during acknowledgement (%s)", async outcome => { const seeded = await seedQueue(); await seedDispatchIdentity(seeded); steerNativeSessionMock.mockImplementationOnce(async () => { - await db.update(heartbeatRuns).set({ status, finishedAt: new Date(), resultJson: { terminalReceipt: "preserve" } }) + await db.update(heartbeatRuns).set({ + ...(outcome === "different task" ? { contextSnapshot: { issueId: randomUUID() } } : { status: outcome, finishedAt: new Date() }), + resultJson: { providerReceipt: "preserve" }, + }) .where(eq(heartbeatRuns.id, seeded.runId)); return { turnId: "turn-that-stopped" }; }); @@ -2279,7 +2285,9 @@ describeEmbeddedPostgres("issue queued-comment routes", () => { expect(wake.status).toBe("deferred_issue_execution"); expect((wake.payload as any)._paperclipWakeContext.wakeCommentIds).toEqual(seeded.commentIds); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId)); - expect(run.resultJson).toEqual({ terminalReceipt: "preserve" }); + expect(run.resultJson).toEqual({ providerReceipt: "preserve" }); + const [identity] = await db.select().from(runIdentityContexts).where(eq(runIdentityContexts.messageId, seeded.commentIds[0])); + expect(identity?.status).toBe("pending"); const activity = await db.select().from(activityLog).where(eq(activityLog.action, "issue.queued_comment_steered")); expect(activity).toHaveLength(0); }); diff --git a/server/src/modules/wake-queue/adapters/queued-comment-board-postgres.ts b/server/src/modules/wake-queue/adapters/queued-comment-board-postgres.ts index 58ef032d1e4..f8fb5bf18e8 100644 --- a/server/src/modules/wake-queue/adapters/queued-comment-board-postgres.ts +++ b/server/src/modules/wake-queue/adapters/queued-comment-board-postgres.ts @@ -1,11 +1,14 @@ import { agentWakeupRequests, agents, heartbeatRuns, issueComments, issues, type Db } from "@paperclipai/db"; import type { IssueQueuedCommentQueue } from "@paperclipai/shared"; import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm"; -import { conflict, notFound } from "../../../errors.js"; +import { assertQueuedCommentInteractionDelivery, recheckQueuedCommentInteractionDelivery } from "../../../services/agent-interaction-delivery.js"; +import { deliverSteeredMessage, steeringDeliveryIsUncertain } from "../../../services/queued-comment-steering.js"; +import { acceptSteeredIdentity, rejectSteeredIdentity, reserveSteeredIdentity, type RunIdentityContext } from "../../../services/run-identity.js"; import { buildQueuedCommentQueueSnapshot, decideQueuedCommentQueueSteering, queuedCommentIdsFromWakePayload, + withQueuedCommentIdsInWakePayload, } from "../../../services/issue-queued-comment-queue.js"; import { firstTurnClaimedCommentIds, @@ -16,13 +19,15 @@ import { } from "../../../services/native-runtime/first-turn-queued-input.js"; import { getNativeSessionSteeringState } from "../../../services/native-runtime/native-session-executor.js"; import { queuedInteractionId, readQueuedInteractionResponse, type QueuedInteractionResponse } from "../../../services/queued-interaction-response.js"; -import type { QueuedCommentActor } from "../application/queued-comment-ports.js"; +import type { QueuedCommentActor, QueuedCommentSteeringPort, SteerQueuedCommentInput } from "../application/queued-comment-ports.js"; +import { QueuedCommentMutationError } from "../application/queued-comment-use-cases.js"; import { parseObject } from "../domain/values.js"; type IssueQueueDb = Db | Parameters[0]>[0]; type IssueQueueTx = Parameters[0]>[0]; type IssueQueueWake = typeof agentWakeupRequests.$inferSelect; type IssueQueueRun = typeof heartbeatRuns.$inferSelect; +type IssueRow = typeof issues.$inferSelect; type IssueQueueState = { wake: Pick; @@ -248,21 +253,18 @@ export async function readBoardQueuedComments(input: { return queue; } -export async function lockBoardQueuedComments(input: { +type BoardQueueTarget = Pick & { tx: IssueQueueTx; - issue: { - id: string; - companyId: string; - assigneeAgentId: string | null; - executionRunId?: string | null; - }; - actor: QueuedCommentActor; - queueId: string; targetRunId?: string; allowStoppedTarget?: boolean; - /** Steering must allow PRP event ingestion to update the run before ACK. */ - lockRun?: boolean; -}) { +}; + +export async function lockBoardQueuedComments(input: BoardQueueTarget) { + const { issue, wake } = await lockBoardQueueRows(input); + return readLockedBoardQueue({ ...input, issue }, wake); +} + +async function lockBoardQueueRows(input: Pick) { const [currentIssue] = await input.tx .select() .from(issues) @@ -273,8 +275,7 @@ export async function lockBoardQueuedComments(input: { ), ) .for("update"); - if (!currentIssue) throw notFound("Issue not found"); - input.issue = currentIssue; + if (!currentIssue) throw new QueuedCommentMutationError("queued_comment_issue_not_found", "Issue not found"); const wake = await input.tx .select() .from(agentWakeupRequests) @@ -282,14 +283,28 @@ export async function lockBoardQueuedComments(input: { and( eq(agentWakeupRequests.id, input.queueId), eq(agentWakeupRequests.companyId, input.issue.companyId), - input.issue.assigneeAgentId - ? eq(agentWakeupRequests.agentId, input.issue.assigneeAgentId) + currentIssue.assigneeAgentId + ? eq(agentWakeupRequests.agentId, currentIssue.assigneeAgentId) : undefined, ), ) .for("update") .limit(1) .then((rows) => rows[0] ?? null); + return { issue: currentIssue, wake }; +} + +function runTargetsIssue(run: IssueQueueRun, issueId: string) { + const context = parseObject(run.contextSnapshot); + return context.issueId === issueId || context.taskId === issueId; +} + +async function readLockedBoardQueue( + input: BoardQueueTarget & { issue: IssueRow }, + wake: IssueQueueWake | null, + /** Steer reads the scoped run without locking it; Interrupt still uses its existing run lock. */ + steeringRun?: IssueQueueRun | null, +) { if ( !wake || parseObject(wake.payload).issueId !== input.issue.id || @@ -299,9 +314,7 @@ export async function lockBoardQueuedComments(input: { (await readQueuedInteractionResponse(input.tx as unknown as Db, input.issue.companyId, input.issue.id, wake.payload)) ?.toolActionOutcome) ) { - throw conflict("The queued message is no longer pending", { - code: "queued_comment_not_pending", - }); + throw new QueuedCommentMutationError("queued_comment_not_pending", "The queued message is no longer pending"); } let state: IssueQueueState["state"]; @@ -324,9 +337,7 @@ export async function lockBoardQueuedComments(input: { .limit(1) .then((rows) => rows[0] ?? null); if (!queueRun || queueRun.status !== "queued") { - throw conflict("The queued message is already being dispatched", { - code: "queued_comment_already_dispatching", - }); + throw new QueuedCommentMutationError("queued_comment_already_dispatching", "The queued message is already being dispatched"); } state = "queued"; } else if ( @@ -334,47 +345,22 @@ export async function lockBoardQueuedComments(input: { wake.status === "running" || (wake.runId && (wake.status === "succeeded" || wake.status === "failed")) ) { - throw conflict("The queued message is already being dispatched", { - code: "queued_comment_already_dispatching", - }); + throw new QueuedCommentMutationError("queued_comment_already_dispatching", "The queued message is already being dispatched"); } else { - throw conflict("The queued message is no longer pending", { - code: "queued_comment_not_pending", - }); + throw new QueuedCommentMutationError("queued_comment_not_pending", "The queued message is no longer pending"); } - const activeRunId = - state === "deferred" - ? (input.targetRunId ?? input.issue.executionRunId ?? null) + const activeRunId = state === "deferred" ? (input.targetRunId ?? input.issue.executionRunId) : null; + const activeRun = steeringRun !== undefined + ? activeRunId && steeringRun?.status === "running" ? steeringRun : null + : activeRunId + ? await input.tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, activeRunId), eq(heartbeatRuns.companyId, input.issue.companyId), + input.allowStoppedTarget ? undefined : eq(heartbeatRuns.status, "running"), + )).for("update").limit(1).then(rows => rows[0] ?? null) : null; - const activeRunQuery = activeRunId - ? input.tx - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.id, activeRunId), - eq(heartbeatRuns.companyId, input.issue.companyId), - input.allowStoppedTarget ? undefined : eq(heartbeatRuns.status, "running"), - ), - ) - .$dynamic() - : null; - const activeRun = activeRunQuery - ? await (input.lockRun === false ? activeRunQuery : activeRunQuery.for("update")) - .limit(1).then(rows => rows[0] ?? null) - : null; - if (input.targetRunId) { - const runContext = parseObject(activeRun?.contextSnapshot); - if ( - !activeRun || - (runContext.issueId !== input.issue.id && - runContext.taskId !== input.issue.id) - ) { - throw conflict("The queued message targets a stale run", { - code: "queued_comment_stale_target", - }); - } + if (input.targetRunId && (!activeRun || !runTargetsIssue(activeRun, input.issue.id))) { + throw new QueuedCommentMutationError("queued_comment_stale_target", "The queued message targets a stale run"); } const queueState = { wake, state, queueRun } satisfies IssueQueueState; const queue = await readBoardQueuedComments({ @@ -390,3 +376,93 @@ export async function lockBoardQueuedComments(input: { }); return { activeRun, wake, queueRun, state, queue, queueState }; } + +export function createQueuedCommentSteeringAdapter(db: Db): QueuedCommentSteeringPort { + return { + async reserveIdentity({ issue, queueId, commentId, targetRunId }) { + const [wake] = await db.select({ agentId: agentWakeupRequests.agentId, payload: agentWakeupRequests.payload }) + .from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, issue.companyId), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, + )); + const response = wake ? await readQueuedInteractionResponse(db, issue.companyId, issue.id, wake.payload) : null; + if (wake) await recheckQueuedCommentInteractionDelivery(db, { + companyId: issue.companyId, issueId: issue.id, agentId: wake.agentId, commentId, queueId, + }); + return reserveSteeredIdentity(db, { + companyId: issue.companyId, issueId: issue.id, runId: targetRunId, messageId: commentId, + source: response?.comment.id === commentId ? "interaction" : "comment", + }); + }, + rejectIdentity: (identity) => rejectSteeredIdentity(db, identity), + isDeliveryUncertain: steeringDeliveryIsUncertain, + + withLockedTarget(input, apply) { + return db.transaction(async (tx) => { + const { issue, wake } = await lockBoardQueueRows({ ...input, tx }); + const { companyId } = issue; + const { targetRunId, commentId, actor } = input; + // A spent wake can still identify a durable acknowledgement for a retry. + const runScope = wake && parseObject(wake.payload).issueId === issue.id ? and( + eq(heartbeatRuns.id, targetRunId), eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.agentId, wake.agentId), + ) : undefined; + const [candidate] = runScope ? await tx.select().from(heartbeatRuns).where(runScope) : []; + const run = candidate && runTargetsIssue(candidate, issue.id) ? candidate : null; + const previousAcknowledgements = parseObject(parseObject(run?.resultJson).queuedSteeringAcknowledgements); + return apply({ + previousAcknowledgement: parseObject(previousAcknowledgements[commentId]), + readReplayQueue: () => readBoardQueuedComments({ + executor: tx, issue, actor, activeRun: run?.status === "running" ? run : null, + }), + async readPendingQueue() { + const { queue } = await readLockedBoardQueue({ ...input, tx, issue }, wake, run); + return queue; + }, + assertDeliveryAllowed: () => assertQueuedCommentInteractionDelivery(tx, { + companyId, issueId: issue.id, agentId: run!.agentId, commentId, lock: true, + }), + deliver: (message, identity) => deliverSteeredMessage(db, { + executor: tx, companyId, runId: targetRunId, messageId: commentId, message, identity, + }), + async acceptAcknowledgement({ identity, turnId, remainingIds }) { + if (!runScope || !wake) return null; + // PRP event ingestion needs this row to persist its ACK. Re-read and + // lock it now so concurrent receipts survive the result merge. + const [acknowledgedRun] = await tx.select().from(heartbeatRuns).where(and( + runScope, eq(heartbeatRuns.status, "running"), + )).for("update"); + if (!acknowledgedRun || !runTargetsIssue(acknowledgedRun, issue.id)) return null; + const result = parseObject(acknowledgedRun.resultJson); + if (identity) await acceptSteeredIdentity(tx, identity); + + const now = new Date(); + const wakeScope = and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, companyId)); + let nextWake: IssueQueueWake | null = null; + if (remainingIds.length === 0) { + await tx.update(agentWakeupRequests).set({ status: "cancelled", finishedAt: now, updatedAt: now }).where(wakeScope); + } else { + const [updated] = await tx.update(agentWakeupRequests).set({ + payload: withQueuedCommentIdsInWakePayload(wake.payload, remainingIds), updatedAt: now, + }).where(wakeScope).returning(); + nextWake = updated ?? wake; + } + await tx.update(heartbeatRuns).set({ + resultJson: { + ...result, + queuedSteeringAcknowledgements: { + ...parseObject(result.queuedSteeringAcknowledgements), + [commentId]: { status: "acknowledged", queueId: wake.id, turnId, acknowledgedAt: now.toISOString() }, + }, + }, + updatedAt: now, + }).where(runScope); + return readBoardQueuedComments({ + executor: tx, issue, actor, activeRun: acknowledgedRun, + queueState: nextWake ? { wake: nextWake, state: "deferred", queueRun: null } : null, + }); + }, + }); + }); + }, + }; +} diff --git a/server/src/modules/wake-queue/application/queued-comment-ports.ts b/server/src/modules/wake-queue/application/queued-comment-ports.ts index d3a4f48f01c..e888f52a914 100644 --- a/server/src/modules/wake-queue/application/queued-comment-ports.ts +++ b/server/src/modules/wake-queue/application/queued-comment-ports.ts @@ -179,3 +179,34 @@ export interface QueuedCommentIssueLockWriter { fn: (locked: LockedQueuedCommentState, transaction: QueuedCommentQueueTransaction) => Promise, ): Promise; } + +export type SteerQueuedCommentInput = { + issue: Pick; + actor: QueuedCommentActor; + queueId: string; + commentId: string; + targetRunId: string; + revision: string; +}; + +interface QueuedCommentSteeringTransaction { + previousAcknowledgement: { status?: unknown; queueId?: unknown; turnId?: unknown }; + readReplayQueue(): Promise; + readPendingQueue(): Promise; + assertDeliveryAllowed(): Promise; + deliver(message: string, identity: Identity | null): Promise<{ turnId: string }>; + /** Locks the run after provider acknowledgement and commits the receipt and queue change together. */ + acceptAcknowledgement(input: { + identity: Identity | null; + turnId: string; + remainingIds: string[]; + }): Promise; +} + +export interface QueuedCommentSteeringPort { + /** Persists the identity independently so a late provider acknowledgement can survive rollback. */ + reserveIdentity(input: SteerQueuedCommentInput): Promise; + rejectIdentity(identity: Identity): Promise; + isDeliveryUncertain(error: unknown, attempted: boolean): boolean; + withLockedTarget(input: SteerQueuedCommentInput, apply: (tx: QueuedCommentSteeringTransaction) => Promise): Promise; +} diff --git a/server/src/modules/wake-queue/application/queued-comment-use-cases.ts b/server/src/modules/wake-queue/application/queued-comment-use-cases.ts index 68fe32cbc80..292a8d1eb5a 100644 --- a/server/src/modules/wake-queue/application/queued-comment-use-cases.ts +++ b/server/src/modules/wake-queue/application/queued-comment-use-cases.ts @@ -11,16 +11,21 @@ import type { QueuedCommentQueueSnapshot, QueuedCommentQueueTransaction, QueuedCommentRunRow, + QueuedCommentSteeringPort, + SteerQueuedCommentInput, } from "./queued-comment-ports.js"; export type QueuedCommentMutationErrorCode = | "queued_comment_not_pending" | "queued_comment_already_dispatching" | "queued_comment_stale_queue" + | "queued_comment_stale_target" + | "queued_comment_issue_not_found" + | "queued_response_requires_fresh_session" + | "steering_unsupported" | "queued_comment_revision_conflict" | "queued_comment_order_mismatch"; -/** The route maps this 1:1 onto the `conflict(...)` HTTP error it threw before this move, using `code` and `message` unchanged. */ export class QueuedCommentMutationError extends Error { constructor( readonly code: QueuedCommentMutationErrorCode, @@ -39,7 +44,7 @@ export class QueuedCommentMutationForbiddenError extends Error { } } -function requireMutationTarget(queue: QueuedCommentQueueSnapshot, queueId: string, revision: string): void { +export function requireQueuedCommentMutationTarget(queue: QueuedCommentQueueSnapshot, queueId: string, revision: string): void { if (queue.queueId !== queueId) { throw new QueuedCommentMutationError("queued_comment_stale_queue", "The queued message targets a stale queue"); } @@ -48,6 +53,54 @@ function requireMutationTarget(queue: QueuedCommentQueueSnapshot, queueId: strin } } +export type SteerQueuedCommentResult = { + queue: QueuedCommentQueueSnapshot; + turnId: string | null; + duplicate: boolean; +}; + +export function createSteerQueuedComment(steering: QueuedCommentSteeringPort) { + return async function steerQueuedComment(input: SteerQueuedCommentInput): Promise { + const identity = await steering.reserveIdentity(input); + let attempted = false; + try { + return await steering.withLockedTarget(input, async (tx) => { + const prior = tx.previousAcknowledgement; + if (prior.status === "acknowledged" && prior.queueId === input.queueId) { + return { queue: await tx.readReplayQueue(), duplicate: true, + turnId: typeof prior.turnId === "string" ? prior.turnId : null }; + } + + const queue = await tx.readPendingQueue(); + requireQueuedCommentMutationTarget(queue, input.queueId, input.revision); + if (queue.protocol !== "paperclip_runner_v1") { + throw new QueuedCommentMutationError("steering_unsupported", "This runner does not support same-turn steering"); + } + const entry = queue.entries.find((candidate) => candidate.comment.id === input.commentId); + if (!entry) { + throw new QueuedCommentMutationError("queued_comment_not_pending", "The queued message is no longer pending"); + } + if (entry.source?.requiresFreshSession) { + throw new QueuedCommentMutationError("queued_response_requires_fresh_session", + "This approval needs a fresh turn. Interrupt or wait for the current turn to finish."); + } + await tx.assertDeliveryAllowed(); + attempted = true; + const { turnId } = await tx.deliver(entry.comment.body, identity); + const remainingIds = queue.entries.map(({ comment }) => comment.id).filter((id) => id !== input.commentId); + const updatedQueue = await tx.acceptAcknowledgement({ identity, turnId, remainingIds }); + if (!updatedQueue) { + throw new QueuedCommentMutationError("queued_comment_stale_target", "The queued message targets a stopped run. Your message is still queued."); + } + return { queue: updatedQueue, turnId, duplicate: false }; + }); + } catch (error) { + if (identity && !steering.isDeliveryUncertain(error, attempted)) await steering.rejectIdentity(identity); + throw error; + } + }; +} + async function updateQueueRunCommentIdsGuarded( tx: QueuedCommentQueueTransaction, input: { queueRun: QueuedCommentRunRow | null; ids: string[]; updatedAt: Date }, @@ -85,7 +138,7 @@ export function createEditQueuedComment(deps: { issueLock: QueuedCommentIssueLoc return deps.issueLock.withLockedQueue( { issue: input.issue, actor: input.actor, queueId: input.queueId }, async (locked, tx) => { - requireMutationTarget(locked.queue, input.queueId, input.revision); + requireQueuedCommentMutationTarget(locked.queue, input.queueId, input.revision); const entry = locked.queue.entries.find((candidate) => candidate.comment.id === input.commentId); if (!entry) { throw new QueuedCommentMutationError("queued_comment_not_pending", "The queued message is no longer pending"); @@ -163,7 +216,7 @@ export function createReorderQueuedComments(deps: { issueLock: QueuedCommentIssu return deps.issueLock.withLockedQueue( { issue: input.issue, actor: input.actor, queueId: input.queueId }, async (locked, tx) => { - requireMutationTarget(locked.queue, input.queueId, input.revision); + requireQueuedCommentMutationTarget(locked.queue, input.queueId, input.revision); const currentIds = locked.queue.entries.map((entry) => entry.comment.id); const reorderDecision = decideQueuedCommentReorder({ currentIds, orderedIds: input.orderedCommentIds }); @@ -249,7 +302,7 @@ export function createDiscardQueuedComment(deps: { issueLock: QueuedCommentIssue { issue: input.issue, actor: input.actor, queueId: input.queueId }, async (locked, tx) => { if (input.revision !== undefined) { - requireMutationTarget(locked.queue, input.queueId, input.revision); + requireQueuedCommentMutationTarget(locked.queue, input.queueId, input.revision); } const entry = locked.queue.entries.find((candidate) => candidate.comment.id === input.commentId); diff --git a/server/src/modules/wake-queue/index.ts b/server/src/modules/wake-queue/index.ts index 0ab592c4b47..6c7852e85cc 100644 --- a/server/src/modules/wake-queue/index.ts +++ b/server/src/modules/wake-queue/index.ts @@ -6,7 +6,7 @@ import { type IssueAdmissionTarget, } from "./adapters/postgres.js"; import { createQueuedCommentIssueLockWriter } from "./adapters/queued-comment-postgres.js"; -import { lockBoardQueuedComments, readBoardQueuedComments } from "./adapters/queued-comment-board-postgres.js"; +import { createQueuedCommentSteeringAdapter, lockBoardQueuedComments, readBoardQueuedComments } from "./adapters/queued-comment-board-postgres.js"; import type { QueuedCommentQueuePostgresAdapterDeps } from "./adapters/queued-comment-postgres.js"; import { createFreshIssueRunAdmission } from "./application/fresh-issue-run.js"; import { admitIssueWake, type IssueWakeAdmissionEffects, type IssueWakeAdmissionOutcome } from "./application/issue-wake-admission.js"; @@ -15,6 +15,7 @@ import { createDiscardQueuedComment, createEditQueuedComment, createReorderQueuedComments, + createSteerQueuedComment, } from "./application/queued-comment-use-cases.js"; import type { IssueSnapshot, @@ -42,6 +43,7 @@ export type { ReleaseIssueExecutionInput } from "./application/use-cases.js"; export { QueuedCommentMutationError, QueuedCommentMutationForbiddenError, + requireQueuedCommentMutationTarget, } from "./application/queued-comment-use-cases.js"; export type { DiscardQueuedCommentInput, @@ -125,9 +127,8 @@ export function createWakeQueue(db: Db, deps: WakeQueueDeps) { export type WakeQueue = ReturnType; /** - * Composes the three queued-comment queue mutations (edit, reorder, - * discard): the Postgres adapter, which owns the one transaction each - * mutation runs in, and the three use cases. This is a separate factory + * Composes the queued-comment reads and mutations with their Postgres + * adapters. This is a separate factory * from `createWakeQueue` because these mutations need none of the release * or admission host callbacks -- only the small set of comment-reference * and external-object sync callbacks in `deps`, which a caller outside @@ -138,6 +139,7 @@ export function createQueuedCommentQueue(db: Db, deps: QueuedCommentQueuePostgre return { readBoardQueue: readBoardQueuedComments, lockBoardQueue: lockBoardQueuedComments, + steerQueuedComment: createSteerQueuedComment(createQueuedCommentSteeringAdapter(db)), editQueuedComment: createEditQueuedComment({ issueLock }), reorderQueuedComments: createReorderQueuedComments({ issueLock }), discardQueuedComment: createDiscardQueuedComment({ issueLock }), diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index b5b110fa75f..f77a4a36cd3 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -1,6 +1,5 @@ import { assertAgentInteraction } from "../services/agent-interaction-policy.js"; import { isAppQuestionCardKey } from "../services/mcp-elicitation.js"; -import { assertQueuedCommentInteractionDelivery, recheckQueuedCommentInteractionDelivery } from "../services/agent-interaction-delivery.js"; import { listStalledDependencies } from "../services/recovery/blocker-stall-wakeups.js"; import { confirmMcpAttachmentCleanup, recordMcpAttachmentCleanup, type McpAttachmentCleanup } from "../services/public-mcp/upload-cleanup.js"; import { notifyDeferredExecutionWork } from "../services/deferred-execution-notifications.js"; @@ -60,7 +59,7 @@ import { retryNativeWorkspaceExport } from "../services/native-runtime/native-wo import { deferredWakeJoinsFirstTurn, firstTurnTakesAllQueuedComments, runAcceptsFirstTurnQueuedInput } from "../services/native-runtime/first-turn-queued-input.js"; import { reconstructNativeWorkspace, unreconciledNativeExportPredicate } from "../services/native-runtime/native-workspace-reconstruction.js"; import { reconstructWorkspaceSchema } from "@paperclipai/shared"; -import { readQueuedInteractionResponse, hasQueuedInteractionResponse } from "../services/queued-interaction-response.js"; +import { hasQueuedInteractionResponse } from "../services/queued-interaction-response.js"; import { deliverConversationComments, heldConversationComments, isConversation } from "../services/agent-conversations.js"; import { assertOnboardingFirstTaskDependenciesAllowed } from "../services/onboarding-first-task-admission.js"; import { resolveChatTaskAssignment } from "../services/chat-task-assignment.js"; @@ -72,11 +71,6 @@ import { validateExecutionReconciliation, markExecutionReconciliation, } from "../services/execution-recovery-resolution.js"; -import { - reserveSteeredIdentity, - acceptSteeredIdentity, - rejectSteeredIdentity, -} from "../services/run-identity.js"; import { createHash, randomUUID } from "node:crypto"; import { Router, type Request, type Response } from "express"; import multer from "multer"; @@ -194,7 +188,6 @@ import { type IssueThreadInteractionCanonicalResolverPolicy, type IssueComment, type IssueCommentPresentation, - type IssueQueuedCommentQueue, type IssueWatchdogDiscoveryKind, type ProjectWorkspace, type SourceTrustMetadata, @@ -259,6 +252,7 @@ import { questionResponseDeliveryService } from "../services/question-response-d import { emitAgentTaskRunById } from "../services/agent-task-run-telemetry.js"; import { createQueuedCommentQueue, + requireQueuedCommentMutationTarget, QueuedCommentMutationError, QueuedCommentMutationForbiddenError, type QueuedCommentIssueContext, @@ -428,9 +422,7 @@ import { import { buildQueuedCommentQueueSnapshot, queuedCommentIdsFromWakePayload, - withQueuedCommentIdsInWakePayload, } from "../services/issue-queued-comment-queue.js"; -import { deliverSteeredMessage, steeringDeliveryIsUncertain } from "../services/queued-comment-steering.js"; const MAX_ISSUE_COMMENT_LIMIT = 500; const updateIssueRouteSchema = updateIssueSchema.extend({ @@ -7702,23 +7694,6 @@ export function issueRoutes( type IssueQueueDb = Db | Parameters[0]>[0]; - function assertQueueMutationTarget(input: { - queue: IssueQueuedCommentQueue; - queueId: string; - revision: string; - }) { - if (input.queue.queueId !== input.queueId) { - throw conflict("The queued message targets a stale queue", { - code: "queued_comment_stale_queue", - }); - } - if (input.queue.revision !== input.revision) { - throw conflict("The queued messages changed in another session", { - code: "queued_comment_revision_conflict", - }); - } - } - function operatorInterruptCancelOptions(input: { issueId: string; actor: ReturnType }) { return { errorCode: "operator_interrupted", @@ -17446,11 +17421,15 @@ export function issueRoutes( /** Maps a wake-queue queued-comment mutation error onto the same HTTP error the route threw before this mutation moved into the module. */ function throwForQueuedCommentMutationError(error: unknown): never { if (error instanceof QueuedCommentMutationError) { + if (error.code === "queued_comment_issue_not_found") throw notFound(error.message); throw conflict(error.message, { code: error.code }); } if (error instanceof QueuedCommentMutationForbiddenError) { throw forbidden(error.message); } + if (error instanceof NativeSessionSteeringError) { + throw conflict(error.message, { code: error.code, retryable: true }); + } throw error; } @@ -17603,12 +17582,12 @@ export function issueRoutes( } } const actor = getActorInfo(req); - const interruptPlan = await db.transaction(async (tx): Promise<{ firstTurnRunId: string } | { runToInterrupt: string | null }> => { + const interruptPlan = await runQueuedCommentMutation(() => db.transaction(async (tx): Promise<{ firstTurnRunId: string } | { runToInterrupt: string | null }> => { const locked = await queuedCommentQueue.lockBoardQueue({ tx, issue, actor, queueId: req.body.queueId, targetRunId: req.body.targetRunId ?? undefined, allowStoppedTarget: true, }); - assertQueueMutationTarget({ queue: locked.queue, queueId: req.body.queueId, revision: req.body.revision }); + requireQueuedCommentMutationTarget(locked.queue, req.body.queueId, req.body.revision); // Explicit Interrupt is also the fallback when a native turn cannot // accept steering. Cancellation and ordinary admission still own Stop // confirmation, cleanup, and delivery; this click only records intent. @@ -17642,7 +17621,7 @@ export function issueRoutes( updatedAt: new Date(), }).where(eq(agentWakeupRequests.id, locked.wake.id)); return { runToInterrupt: locked.activeRun?.status === "running" ? locked.activeRun.id : null }; - }); + })); const firstTurnRunId = "firstTurnRunId" in interruptPlan ? interruptPlan.firstTurnRunId : null; const runToInterrupt = "runToInterrupt" in interruptPlan ? interruptPlan.runToInterrupt : null; // Never hold the issue lock while joining the adapter. Queue edits and @@ -17693,243 +17672,11 @@ export function issueRoutes( const decision = await decideIssueAccess(req, issue, "issue:comment"); if (!decision.allowed) throw forbidden(decision.explanation, authorizationDeniedDetails(decision)); const actor = getActorInfo(req); - const responseWake = await db.select().from(agentWakeupRequests).where(and( - eq(agentWakeupRequests.id, req.body.queueId), eq(agentWakeupRequests.companyId, issue.companyId), - )).then(rows => rows[0]); - const response = responseWake ? await readQueuedInteractionResponse(db, issue.companyId, issue.id, responseWake.payload) : null; - if (responseWake) await recheckQueuedCommentInteractionDelivery(db, { - companyId: issue.companyId, issueId: issue.id, agentId: responseWake.agentId, - commentId, queueId: responseWake.id, - }); - const steeringIdentity = await reserveSteeredIdentity(db, { - companyId: issue.companyId, - runId: req.body.targetRunId, - issueId: issue.id, - messageId: commentId, - source: response?.comment.id === commentId ? "interaction" : "comment", - }); - let steeringDeliveryAttempted = false; - let acknowledgedTurnId: string | null = null; - let duplicate = false; - let queue: IssueQueuedCommentQueue; - try { - queue = await db.transaction(async (tx) => { - // A client can lose the successful response after the final queued - // message cancels its wake. Lock the original queue and target run - // first so that the persisted acknowledgement remains a durable - // idempotency record even when no pending queue remains. - await tx - .select({ id: issueRows.id }) - .from(issueRows) - .where( - and( - eq(issueRows.id, issue.id), - eq(issueRows.companyId, issue.companyId), - ), - ) - .for("update"); - const retryWake = await tx - .select() - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.id, req.body.queueId), - eq(agentWakeupRequests.companyId, issue.companyId), - issue.assigneeAgentId - ? eq(agentWakeupRequests.agentId, issue.assigneeAgentId) - : undefined, - ), - ) - .for("update") - .limit(1) - .then((rows) => rows[0] ?? null); - const retryRun = - retryWake && readObject(retryWake.payload).issueId === issue.id - ? await tx - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.id, req.body.targetRunId), - eq(heartbeatRuns.companyId, issue.companyId), - eq(heartbeatRuns.agentId, retryWake.agentId), - ), - ) - .limit(1) - .then((rows) => rows[0] ?? null) - : null; - const retryRunContext = readObject(retryRun?.contextSnapshot); - const retryRunResult = readObject(retryRun?.resultJson); - const retryAcknowledgements = readObject( - retryRunResult.queuedSteeringAcknowledgements, - ); - const retryAcknowledgement = readObject( - retryAcknowledgements[commentId], - ); - if ( - retryRun && - (retryRunContext.issueId === issue.id || - retryRunContext.taskId === issue.id) && - retryAcknowledgement.status === "acknowledged" && - retryAcknowledgement.queueId === req.body.queueId - ) { - duplicate = true; - acknowledgedTurnId = - typeof retryAcknowledgement.turnId === "string" - ? retryAcknowledgement.turnId - : null; - return queuedCommentQueue.readBoardQueue({ - executor: tx, - issue, - activeRun: retryRun.status === "running" ? retryRun : null, - actor, - }); - } - - const locked = await queuedCommentQueue.lockBoardQueue({ - tx, - issue, - actor, - queueId: req.body.queueId, - targetRunId: req.body.targetRunId, - lockRun: false, - }); - if (!locked.activeRun) { - throw conflict("The queued message targets a stale run", { - code: "queued_comment_stale_target", - }); - } - const runResult = readObject(locked.activeRun.resultJson); - const acknowledgements = readObject( - runResult.queuedSteeringAcknowledgements, - ); - const priorAcknowledgement = readObject(acknowledgements[commentId]); - if ( - priorAcknowledgement.status === "acknowledged" && - priorAcknowledgement.queueId === req.body.queueId - ) { - duplicate = true; - acknowledgedTurnId = - typeof priorAcknowledgement.turnId === "string" - ? priorAcknowledgement.turnId - : null; - return queuedCommentQueue.readBoardQueue({ - executor: tx, - issue, - activeRun: locked.activeRun, - actor, - queueState: locked.queueState, - }); - } - assertQueueMutationTarget({ - queue: locked.queue, - queueId: req.body.queueId, - revision: req.body.revision, - }); - if (locked.queue.protocol !== "paperclip_runner_v1") { - throw conflict("This runner does not support same-turn steering", { - code: "steering_unsupported", - }); - } - const entry = locked.queue.entries.find( - (candidate) => candidate.comment.id === commentId, - ); - if (!entry) { - throw conflict("The queued message is no longer pending", { - code: "queued_comment_not_pending", - }); - } - - if (entry.source?.requiresFreshSession) { - throw conflict("This approval needs a fresh turn. Interrupt or wait for the current turn to finish.", { - code: "queued_response_requires_fresh_session", - }); - } - await assertQueuedCommentInteractionDelivery(tx, { - companyId: issue.companyId, issueId: issue.id, agentId: locked.activeRun.agentId, - commentId, lock: true, - }); - steeringDeliveryAttempted = true; - const acknowledgement = await deliverSteeredMessage(db, { - executor: tx, companyId: issue.companyId, runId: locked.activeRun.id, - messageId: commentId, message: entry.comment.body, identity: steeringIdentity, - }); - // PRP ingestion allocates event sequences on this row. Locking it - // while awaiting the provider prevents its durable ACK from arriving. - // Re-read under lock after ACK so concurrent run receipts are retained. - const [acknowledgedRun] = await tx.select().from(heartbeatRuns) - .where(eq(heartbeatRuns.id, locked.activeRun.id)).for("update"); - if (!acknowledgedRun || acknowledgedRun.status !== "running") { - throw conflict("The queued message targets a stopped run. Your message is still queued.", { - code: "queued_comment_stale_target", - }); - } - const acknowledgedResult = readObject(acknowledgedRun.resultJson); - if (steeringIdentity) - await acceptSteeredIdentity(tx, steeringIdentity); - acknowledgedTurnId = acknowledgement.turnId; - const remainingIds = locked.queue.entries - .map((candidate) => candidate.comment.id) - .filter((candidateId) => candidateId !== commentId); - const now = new Date(); - const nextWake = - remainingIds.length === 0 - ? await tx - .update(agentWakeupRequests) - .set({ status: "cancelled", finishedAt: now, updatedAt: now }) - .where(eq(agentWakeupRequests.id, locked.wake.id)) - .returning() - .then(() => null) - : await tx - .update(agentWakeupRequests) - .set({ - payload: withQueuedCommentIdsInWakePayload( - locked.wake.payload, - remainingIds, - ), - updatedAt: now, - }) - .where(eq(agentWakeupRequests.id, locked.wake.id)) - .returning() - .then((rows) => rows[0] ?? locked.wake); - await tx - .update(heartbeatRuns) - .set({ - resultJson: { - ...acknowledgedResult, - queuedSteeringAcknowledgements: { - ...readObject(acknowledgedResult.queuedSteeringAcknowledgements), - [commentId]: { - status: "acknowledged", - queueId: req.body.queueId, - turnId: acknowledgement.turnId, - acknowledgedAt: now.toISOString(), - }, - }, - }, - updatedAt: now, - }) - .where(eq(heartbeatRuns.id, locked.activeRun.id)); - return queuedCommentQueue.readBoardQueue({ - executor: tx, - issue, - activeRun: locked.activeRun, - actor, - queueState: nextWake - ? { wake: nextWake, state: "deferred", queueRun: null } - : null, - }); - }); - } catch (error) { - const uncertain = steeringDeliveryIsUncertain(error, steeringDeliveryAttempted); - if (steeringIdentity && !uncertain) - await rejectSteeredIdentity(db, steeringIdentity); - - if (error instanceof NativeSessionSteeringError) { - throw conflict(error.message, { code: error.code, retryable: true }); - } - throw error; - } + const { queue, turnId, duplicate } = await runQueuedCommentMutation(() => + queuedCommentQueue.steerQueuedComment({ + issue, actor, commentId, queueId: req.body.queueId, revision: req.body.revision, targetRunId: req.body.targetRunId, + }), + ); await logActivity(db, { companyId: issue.companyId, actorType: actor.actorType, @@ -17943,7 +17690,7 @@ export function issueRoutes( details: { commentId, targetRunId: req.body.targetRunId, - turnId: acknowledgedTurnId, + turnId, duplicate, }, });