diff --git a/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts b/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts index 008888a1e85..079942f9e60 100644 --- a/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts +++ b/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts @@ -20,11 +20,15 @@ import { issueComments, issueRelations, issueRecoveryActions, + issueThreadInteractions, issueTreeHoldMembers, issueTreeHolds, issues, projects, projectWorkspaces, + toolActionDeliveries, + toolActionRequests, + toolInvocations, workspaceOperations, } from "@paperclipai/db"; import { @@ -74,6 +78,7 @@ vi.mock("../adapters/index.ts", async () => { import { heartbeatService } from "../services/heartbeat.ts"; import { attentionService } from "../services/attention.ts"; import { issueService } from "../services/issues.ts"; +import { toolActionDeliveryService } from "../services/tool-action-delivery.ts"; import { escalateExhaustedIssueReviewPathRecovery } from "../services/recovery/review-path-recovery-escalation.ts"; import { runningProcesses } from "../adapters/index.ts"; import { @@ -125,6 +130,10 @@ describeEmbeddedPostgres("heartbeat resolved dependency wake reconciliation", () await db.delete(issueTreeHolds); await db.delete(issueRelations); await db.delete(issueRecoveryActions); + await db.delete(toolActionDeliveries); + await db.delete(toolActionRequests); + await db.delete(toolInvocations); + await db.delete(issueThreadInteractions); await db.delete(issues); await db.delete(executionWorkspaces); await db.delete(projectWorkspaces); @@ -498,6 +507,121 @@ describeEmbeddedPostgres("heartbeat resolved dependency wake reconciliation", () }); }); + it("keeps a card accepted while its run settles as the review path and queues no recovery run", async () => { + const companyId = randomUUID(); + const agentId = randomUUID(); + const issueId = randomUUID(); + const issuePrefix = `R${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; + await db.insert(companies).values({ + id: companyId, name: "Review Accept Co", issuePrefix, defaultResponsibleUserId: "responsible-user", + requireBoardApprovalForNewAgents: false, + }); + await db.insert(authUsers).values({ id: "responsible-user", name: "Fixture user", email: "responsible-user@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "responsible-user", status: "active", membershipRole: "member" }); + await db.insert(agents).values({ + id: agentId, companyId, name: "Review Agent", role: "engineer", status: "idle", adapterType: "codex_local", + adapterConfig: {}, runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } }, permissions: {}, + }); + await db.insert(issues).values({ + id: issueId, companyId, title: "Create the issue once approved", status: "in_review", priority: "medium", + assigneeAgentId: agentId, responsibleUserId: "responsible-user", issueNumber: 1, identifier: `${issuePrefix}-1`, + }); + const [interaction] = await db.insert(issueThreadInteractions).values({ + companyId, issueId, kind: "request_confirmation", status: "accepted", createdByAgentId: agentId, + payload: { version: 1, prompt: "Run the tool?" }, result: { version: 1, outcome: "accepted" }, resolvedAt: new Date(), + }).returning(); + const [invocation] = await db.insert(toolInvocations).values({ + companyId, agentId, issueId, toolName: "create_issue", approvalState: "approved", status: "pending", + }).returning(); + const [request] = await db.insert(toolActionRequests).values({ + companyId, invocationId: invocation!.id, issueId, interactionId: interaction!.id, status: "executing", + canonicalArgumentsHash: "hash", canonicalArgumentsSummary: { type: "object" } as never, + requestedByAgentId: agentId, decidedByUserId: "responsible-user", decidedAt: new Date(), + }).returning(); + await db.insert(toolActionDeliveries).values({ + actionRequestId: request!.id, companyId, issueId, interactionId: interaction!.id, + }); + + const heartbeat = heartbeatService(db); + const sourceRun = await heartbeat.wakeup(agentId, { + source: "automation", + triggerDetail: "system", + reason: "issue_commented", + payload: { issueId }, + requestedByActorType: "user", + requestedByActorId: "responsible-user", + contextSnapshot: { issueId, taskId: issueId, wakeReason: "issue_commented" }, + }); + expect(sourceRun).not.toBeNull(); + await heartbeat.drainActiveRunExecutions(); + + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, sourceRun!.id)).then((rows) => rows[0]?.status)) + .toBe("succeeded"); + expect(await db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.reason, "issue_review_path_lost"), + ))).toHaveLength(0); + expect(await db.select().from(toolActionDeliveries).where(eq(toolActionDeliveries.actionRequestId, request!.id)) + .then((rows) => rows[0]?.deliveredAt)).toBeNull(); + + await db.update(toolInvocations).set({ runId: sourceRun!.id, status: "succeeded" }).where(eq(toolInvocations.id, invocation!.id)); + await db.update(toolActionRequests).set({ status: "executed", resolvedAt: new Date() }).where(eq(toolActionRequests.id, request!.id)); + expect(await toolActionDeliveryService(db, heartbeat).deliver(request!.id)).toBe(true); + await heartbeat.drainActiveRunExecutions(); + const wakes = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); + expect(wakes.filter((wake) => wake.idempotencyKey === `tool-action-response:${request!.id}`)).toHaveLength(1); + expect(wakes.filter((wake) => wake.reason === "issue_review_path_lost" + && wake.payload?.sourceRunId === sourceRun!.id)).toHaveLength(0); + }); + + it.each(["undelivered", "delivered_with_queued_continuation", "delivered_with_listed_wake", "pending_card", "terminated_assignee", "delivered_and_consumed"] as const)( + "reads the accepted tool review continuation as a review path only while it can still run (%s)", + async (state) => { + const companyId = randomUUID(); + const agentId = randomUUID(); + const issueId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Review Path Co", issuePrefix: `R${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ + id: agentId, companyId, name: "Reviewer", role: "engineer", adapterType: "codex_local", + status: state === "terminated_assignee" ? "terminated" : "idle", + }); + await db.insert(issues).values({ id: issueId, companyId, title: "Review", status: "in_review", assigneeAgentId: agentId }); + const [interaction] = await db.insert(issueThreadInteractions).values({ + companyId, issueId, kind: "request_confirmation", createdByAgentId: agentId, + payload: { version: 1, prompt: "Run the tool?" }, + ...(state === "pending_card" ? { status: "pending" } : { status: "accepted", result: { version: 1, outcome: "accepted" }, resolvedAt: new Date() }), + }).returning(); + const [invocation] = await db.insert(toolInvocations).values({ companyId, agentId, issueId, toolName: "create_issue" }).returning(); + const [request] = await db.insert(toolActionRequests).values({ + companyId, invocationId: invocation!.id, issueId, interactionId: interaction!.id, + status: state === "pending_card" ? "pending" : "executed", + canonicalArgumentsHash: "hash", canonicalArgumentsSummary: { type: "object" } as never, requestedByAgentId: agentId, + }).returning(); + const delivered = state === "delivered_with_queued_continuation" || state === "delivered_with_listed_wake" + || state === "delivered_and_consumed"; + await db.insert(toolActionDeliveries).values({ + actionRequestId: request!.id, companyId, issueId, interactionId: interaction!.id, + deliveredAt: delivered ? new Date() : null, + }); + if (delivered) { + await db.insert(agentWakeupRequests).values({ + companyId, agentId, source: "automation", reason: "issue_commented", + status: state === "delivered_and_consumed" ? "completed" : "queued", + idempotencyKey: `tool-action-response:${request!.id}`, + payload: state === "delivered_with_listed_wake" ? { issueId, taskId: issueId } : {}, + }); + } + + const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0]!); + const attention = (await issueService(db).listReviewAttention(companyId, [issue])).get(issueId)!; + const continuation = state === "undelivered" || state === "delivered_with_queued_continuation" + || state === "delivered_with_listed_wake"; + expect(attention.state).toBe(continuation || state === "pending_card" ? "covered" : "stalled"); + expect(attention.paths.filter((path) => path.kind === "queued_wake")).toHaveLength(continuation ? 1 : 0); + expect(attention.paths.filter((path) => path.kind === "interaction")).toHaveLength(state === "pending_card" ? 1 : 0); + }, + ); + it.each(["monitor", "completed", "reassigned", "newer_run", "other_company"] as const)( "does not escalate a stale review recovery after %s", async (change) => { diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 4fd523545a3..e8a0e7b9470 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -4746,6 +4746,23 @@ async function listIssueReviewAttentionMap( } if (reviewIssues.length === 0) return result; + const activeToolReviewContinuation = sql`lateral ( + select continuation.id, continuation.run_id from ${agentWakeupRequests} continuation + where continuation.company_id = ${companyId} + and continuation.idempotency_key like 'tool-action-response:%' + and continuation.idempotency_key = 'tool-action-response:' || ${toolActionDeliveries.actionRequestId}::text + and continuation.status not in ('skipped', 'failed', 'cancelled') + and ( + continuation.status in ('queued', 'deferred_issue_execution', 'claimed') + or exists ( + select 1 from ${heartbeatRuns} continuation_run + where continuation_run.id = continuation.run_id + and continuation_run.company_id = ${companyId} + and continuation_run.status in ('queued', 'running') + ) + ) + limit 1 + ) active_tool_review_continuation`; const [ agentRows, activeRunRows, @@ -4754,6 +4771,7 @@ async function listIssueReviewAttentionMap( approvalRows, recoveryActionRows, recoveryIssueRows, + toolReviewContinuationRows, ] = await Promise.all([ dbOrTx .select({ @@ -4903,8 +4921,77 @@ async function listIssueReviewAttentionMap( notInArray(issues.status, ["done", "cancelled"]), ), ), + dbOrTx + .select({ + id: toolActionDeliveries.actionRequestId, + companyId: toolActionDeliveries.companyId, + issueId: toolActionDeliveries.issueId, + agentId: toolActionRequests.requestedByAgentId, + status: sql`'queued'`, + reason: sql`'tool_action_review'`, + createdAt: toolActionDeliveries.createdAt, + continuationWakeId: sql`active_tool_review_continuation.id`, + continuationRunId: sql`active_tool_review_continuation.run_id`, + }) + .from(toolActionDeliveries) + .innerJoin( + toolActionRequests, + and( + eq(toolActionRequests.id, toolActionDeliveries.actionRequestId), + eq(toolActionRequests.companyId, companyId), + ), + ) + .innerJoin( + issueThreadInteractions, + and( + eq(issueThreadInteractions.id, toolActionDeliveries.interactionId), + eq(issueThreadInteractions.companyId, companyId), + ), + ) + .innerJoin( + issues, + and( + eq(issues.id, toolActionDeliveries.issueId), + eq(issues.companyId, companyId), + eq(issues.assigneeAgentId, toolActionRequests.requestedByAgentId), + ), + ) + .innerJoin( + agents, + and( + eq(agents.id, toolActionRequests.requestedByAgentId), + eq(agents.companyId, companyId), + ), + ) + .leftJoin(activeToolReviewContinuation, sql`true`) + .where( + and( + eq(toolActionDeliveries.companyId, companyId), + inArray(toolActionRequests.issueId, reviewIds), + or( + and( + isNull(toolActionDeliveries.deliveredAt), + ne(issueThreadInteractions.status, "pending"), + notInArray(agents.status, ["paused", "terminated", "pending_approval"]), + ), + sql`active_tool_review_continuation.id is not null`, + ), + ), + ), ]); + const listedExecutionIds = new Set([ + ...(wakeRows as Array<{ id: string }>).map((row) => row.id), + ...(activeRunRows as Array<{ id: string }>).map((row) => row.id), + ]); + const toolReviewContinuationPaths = toolReviewContinuationRows.filter((row: { + continuationWakeId: string | null; + continuationRunId: string | null; + }) => + !(row.continuationWakeId && listedExecutionIds.has(row.continuationWakeId)) && + !(row.continuationRunId && listedExecutionIds.has(row.continuationRunId)), + ); + const recoveryPaths = [ ...(recoveryActionRows as Array<{ id: string; @@ -4961,7 +5048,7 @@ async function listIssueReviewAttentionMap( relations: [], agents: agentRows, activeRuns: activeRunRows, - queuedWakeRequests: wakeRows, + queuedWakeRequests: [...wakeRows, ...toolReviewContinuationPaths], pendingInteractions: interactionRows, pendingApprovals: approvalRows, externalResponseWaits: await readExternalReviewWaits(reviewIds), @@ -5025,7 +5112,7 @@ async function listIssueReviewAttentionMap( ).map((row) => [row.id, row]), ); const wakeReasonById = new Map( - (wakeRows as Array<{ id: string; reason: string | null }>).map((row) => [ + ([...wakeRows, ...toolReviewContinuationPaths] as Array<{ id: string; reason: string | null }>).map((row) => [ row.id, row.reason, ]),