diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts index 372cd77ba5e..36c2d056587 100644 --- a/server/src/__tests__/issue-recovery-actions.test.ts +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -95,7 +95,7 @@ describe("issueRecoveryActionService", () => { const error: Error & { cause?: unknown } = new Error("Wrapped database failure"); error.cause = kind === "unrelated unique constraint" ? { code: "23505", constraint_name: "other_unique_constraint" } : kind === "other SQLSTATE" ? { code: "23503", constraint_name: constraint } : error; - const query = { from() { return this; }, where() { return this; }, orderBy() { return this; }, + const query = { from() { return this; }, innerJoin() { return this; }, where() { return this; }, orderBy() { return this; }, limit() { return Promise.resolve([]); } }; const fakeDb = { async transaction(fn: (tx: unknown) => Promise): Promise { return fn(fakeDb); }, @@ -106,7 +106,7 @@ describe("issueRecoveryActionService", () => { await expect(issueRecoveryActionService(fakeDb as never).upsertSourceScoped({ companyId: "company", sourceIssueId: "issue", kind: "missing_disposition", cause: "saved-debt", fingerprint: "same-source", nextAction: "Restore existing work." })) .rejects.toBe(error); - expect(fakeDb.select).toHaveBeenCalledOnce(); + expect(fakeDb.select).toHaveBeenCalledTimes(2); expect(fakeDb.insert).toHaveBeenCalledOnce(); }); it("does not reactivate an action resolved between the active read and update", async () => { @@ -118,6 +118,9 @@ describe("issueRecoveryActionService", () => { from() { return this; }, + innerJoin() { + return this; + }, where() { return this; }, diff --git a/server/src/__tests__/issue-recovery-signal-fanout.test.ts b/server/src/__tests__/issue-recovery-signal-fanout.test.ts new file mode 100644 index 00000000000..585f63efcc3 --- /dev/null +++ b/server/src/__tests__/issue-recovery-signal-fanout.test.ts @@ -0,0 +1,133 @@ +import { and, eq } from "drizzle-orm"; +import { expect, it } from "vitest"; +import { issueRecoveryActions, issues } from "@paperclipai/db"; +import { subscribeDatabaseWork } from "@paperclipai/db/work-signals"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { issueRecoveryActionService } from "../services/issue-recovery-actions.js"; +import { notifyIssueGraphRecovery } from "../services/issue-recovery-notifications.js"; +import { recoveryService } from "../services/recovery/service.js"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; + +const TASK_QUEUES = [DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.assignedWorkInvariant, + DELIVERY_QUEUES.resolvedDependencies, DELIVERY_QUEUES.staleIssueLocks]; +const WATCHED = [...TASK_QUEUES, DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, + DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.queuedRunDispatch, DELIVERY_QUEUES.executionDisposition]; + +describeIssueRecovery("issue recovery signal fan-out", () => { + const state = issueRecoveryFixture(); + + function record(owner: object) { + const settled: Array<[string, string[]]> = []; + const stops = WATCHED.map(queue => subscribeDatabaseWork(owner, queue, (event, hints) => { + if (event === "settled") settled.push([queue, [...hints ?? []]]); + })); + return { settled, stop: () => { for (const stop of stops) stop(); } }; + } + + it("publishes one graph hint per task queue and no per-queue backstop twin", async () => { + const f = await state.fixture(); + const [task] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Task", status: "todo" }).returning(); + const recorder = record(state.db); + try { + await state.db.transaction(tx => notifyIssueGraphRecovery(tx, { companyId: f.companyId, issueId: task.id })); + } finally { recorder.stop(); } + const taskHints = recorder.settled.filter(([queue]) => TASK_QUEUES.includes(queue as never)).flatMap(([, hints]) => hints); + expect(taskHints).toHaveLength(TASK_QUEUES.length); + expect(taskHints.every(hint => JSON.parse(hint).graphIssueId === task.id)).toBe(true); + expect(recorder.settled.flatMap(([, hints]) => hints).some(hint => hint.includes("\"backstop\""))).toBe(false); + }); + + it("resolves nothing and signals nothing when the issue has no matching active recovery action", async () => { + const f = await state.fixture(); + const [task] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Done", status: "done" }).returning(); + const [resolved] = await state.db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: task.id, + kind: "stranded_assigned_issue", status: "resolved", ownerType: "agent", ownerAgentId: f.agentId, + cause: "stranded_assigned_issue", fingerprint: "already-resolved", nextAction: "Done", outcome: "restored", + resolvedAt: new Date(), evidence: {} }).returning(); + const recorder = record(state.db); + const actions = issueRecoveryActionService(state.db); + try { + expect(await actions.resolveActiveForIssue({ companyId: f.companyId, sourceIssueId: task.id, actionId: resolved.id, + status: "resolved", outcome: "restored", resolutionNote: "source_terminal" })).toBeNull(); + expect(await actions.resolveActiveForIssue({ companyId: f.companyId, sourceIssueId: task.id, + status: "cancelled", outcome: "cancelled", resolutionNote: "source_state_changed" })).toBeNull(); + } finally { recorder.stop(); } + expect(recorder.settled).toEqual([]); + }); + + it("still resolves an active action and wakes its task graph", async () => { + const f = await state.fixture(); + const [task] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Done", status: "done" }).returning(); + const [active] = await state.db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: task.id, + kind: "stranded_assigned_issue", status: "escalated", ownerType: "agent", ownerAgentId: f.agentId, + cause: "stranded_assigned_issue", fingerprint: "escalated", nextAction: "Restore", evidence: {} }).returning(); + const recorder = record(state.db); + let result; + try { + result = await issueRecoveryActionService(state.db).resolveActiveForIssue({ companyId: f.companyId, sourceIssueId: task.id, + actionId: active.id, status: "resolved", outcome: "restored", resolutionNote: "source_terminal" }); + } finally { recorder.stop(); } + expect(result).toMatchObject({ id: active.id, status: "resolved", outcome: "restored" }); + expect(recorder.settled.filter(([queue]) => queue === DELIVERY_QUEUES.strandedAssignments).flatMap(([, hints]) => hints) + .map(hint => JSON.parse(hint))).toContainEqual({ companyId: f.companyId, graphIssueId: task.id }); + }); + + async function blockedAttention(companyId: string, sourceIssueId: string, agentId: string, fingerprint = "native-controller-space:run:digest") { + return issueRecoveryActionService(state.db).upsertSourceScoped({ companyId, sourceIssueId, kind: "active_run_watchdog", + ownerType: "board", returnOwnerAgentId: agentId, cause: "native_restart_recovery_blocked", fingerprint, + evidence: { runId: "run", reason: "controller_process_space_unverified", executorStopEvidence: "unverified" }, + nextAction: "Wait for the previous controller to stop" }); + } + + it("does not re-open attention that recovery settled because its task finished", async () => { + const f = await state.fixture(); + const [task] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Cancelled", status: "cancelled" }).returning(); + const recovery = recoveryService(state.db, { enqueueWakeup: async () => null }); + await blockedAttention(f.companyId, task.id, f.agentId); + await recovery.reconcileActiveRecoveryActions({ companyId: f.companyId }); + const recorder = record(state.db); + try { + for (let cycle = 0; cycle < 5; cycle++) { + const surfaced = await blockedAttention(f.companyId, task.id, f.agentId); + expect(surfaced.status).toBe("resolved"); + await recovery.reconcileActiveRecoveryActions({ companyId: f.companyId }); + } + } finally { recorder.stop(); } + const rows = await state.db.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, f.companyId), eq(issueRecoveryActions.sourceIssueId, task.id))); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ status: "resolved", outcome: "restored", resolutionNote: "source_terminal" }); + expect(recorder.settled.filter(([queue]) => queue === DELIVERY_QUEUES.strandedAssignments + || queue === DELIVERY_QUEUES.activeRecoveryActions)).toEqual([]); + }); + + it("settles a finished task's attention without waking run dispatch or its task graph", async () => { + const f = await state.fixture(); + const [task] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Done", status: "done" }).returning(); + await blockedAttention(f.companyId, task.id, f.agentId); + const recorder = record(state.db); + try { + expect(await recoveryService(state.db, { enqueueWakeup: async () => null }).reconcileActiveRecoveryActions({ companyId: f.companyId })) + .toMatchObject({ resolved: 1 }); + } finally { recorder.stop(); } + expect(recorder.settled.filter(([queue]) => queue !== DELIVERY_QUEUES.executionDisposition) + .filter(([queue]) => [...TASK_QUEUES, DELIVERY_QUEUES.queuedRunDispatch, DELIVERY_QUEUES.activeRecoveryActions].includes(queue as never))) + .toEqual([]); + }); + + it("surfaces attention again once the task is reopened, and for a new controller authority", async () => { + const f = await state.fixture(); + const [task] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Cancelled", status: "cancelled" }).returning(); + const recovery = recoveryService(state.db, { enqueueWakeup: async () => null }); + await blockedAttention(f.companyId, task.id, f.agentId); + await recovery.reconcileActiveRecoveryActions({ companyId: f.companyId }); + expect((await blockedAttention(f.companyId, task.id, f.agentId, "native-controller-space:run:renewed")).status).toBe("active"); + await recovery.reconcileActiveRecoveryActions({ companyId: f.companyId }); + await state.db.update(issues).set({ status: "todo" }).where(eq(issues.id, task.id)); + const reopened = await blockedAttention(f.companyId, task.id, f.agentId); + expect(reopened.status).toBe("active"); + await recovery.reconcileActiveRecoveryActions({ companyId: f.companyId }); + expect((await state.db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, reopened.id)))[0].status) + .toBe("active"); + }); +}); diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index f9509f33210..9fb786b91fc 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -6,7 +6,7 @@ import { writeExecutionControl } from "./execution-control-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { and, desc, eq, inArray, isNull, isNotNull, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, workspaceOperations } from "@paperclipai/db"; +import { heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, workspaceOperations } from "@paperclipai/db"; import { isUuidLike } from "@paperclipai/shared"; import type { IssueRecoveryAction, @@ -86,8 +86,11 @@ export type ResolveIssueRecoveryActionInput = { status: Extract; outcome: IssueRecoveryActionOutcome; resolutionNote?: string | null; + sourceTerminal?: boolean; }; +export const SOURCE_TERMINAL_RESOLUTION = "source_terminal"; + /** A spent saved-work recovery is described from its current state. Text saved * by an older release could ask a person to inspect the run's environment. */ function readModelNextAction(row: IssueRecoveryActionRow): string { @@ -390,6 +393,21 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const existing = await getActiveForIssue(input.companyId, input.sourceIssueId); const now = new Date(); if (existing && input.createOnly) return existing; + if (!existing) { + const [source] = await db.select({ status: issues.status }).from(issues) + .where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.sourceIssueId))).limit(1); + if (source && (source.status === "done" || source.status === "cancelled")) { + const [settled] = await db.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, input.companyId), + eq(issueRecoveryActions.sourceIssueId, input.sourceIssueId), + eq(issueRecoveryActions.status, "resolved"), + eq(issueRecoveryActions.cause, input.cause), + eq(issueRecoveryActions.fingerprint, input.fingerprint), + eq(issueRecoveryActions.resolutionNote, SOURCE_TERMINAL_RESOLUTION), + )).limit(1); + if (settled) return toReadModel(settled); + } + } const ownerType = input.ownerType ?? (input.ownerAgentId ? "agent" : "board"); if (existing) { // A distinct failure identity must not overwrite the active action of a @@ -619,12 +637,11 @@ export function issueRecoveryActionService(db: DbOrTransaction) { predicates.push(eq(issueRecoveryActions.fingerprint, input.fingerprint)); } - const [updated] = await writeExecutionControl(dbOrTx, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => { - const pending = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) - .where(and(...predicates)).limit(1); - if (!pending.length) return []; - // Resolving another attention releases this issue's preserved Native - // waiter to one normal inspection. No recurring monitor or new turn. + const [candidate] = await dbOrTx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) + .where(and(...predicates)).limit(1); + if (!candidate) return null; + + const releaseWaiter = async (tx: DbOrTransaction) => { const [waiter] = await tx.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations) .innerJoin(heartbeatRuns, eq(heartbeatRuns.id, nativeRunFinalizations.runId)).where(and( eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.issueId, input.sourceIssueId), @@ -632,6 +649,29 @@ export function issueRecoveryActionService(db: DbOrTransaction) { isNotNull(nativeRunFinalizations.leaseOwner), inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted"]), )).limit(1); if (waiter) await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionFinalization); + }; + + if (input.sourceTerminal) { + const [settled] = await dbOrTx.transaction(async tx => { + await releaseWaiter(tx as unknown as DbOrTransaction); + await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); + return tx.update(issueRecoveryActions) + .set({ status: input.status, outcome: input.outcome, resolutionNote: input.resolutionNote ?? null, resolvedAt: now, updatedAt: now }) + .where(and(...predicates, sql`exists (select 1 from ${issues} where ${issues.id} = ${issueRecoveryActions.sourceIssueId} + and ${issues.companyId} = ${issueRecoveryActions.companyId} and ${issues.status} in ('done', 'cancelled'))`)) + .returning(); + }); + return settled ? toReadModel(settled) : null; + } + + const [updated] = await writeExecutionControl(dbOrTx, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => { + const pending = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) + .where(and(...predicates)).limit(1); + if (!pending.length) return []; + // Resolving another attention releases this issue's preserved Native + // waiter to one normal inspection. No recurring monitor or new turn. + await releaseWaiter(tx as unknown as DbOrTransaction); await notifyIssueGraphRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); diff --git a/server/src/services/issue-recovery-notifications.ts b/server/src/services/issue-recovery-notifications.ts index ec3770b7c4d..08751b36220 100644 --- a/server/src/services/issue-recovery-notifications.ts +++ b/server/src/services/issue-recovery-notifications.ts @@ -26,6 +26,10 @@ export async function notifyIssueRecovery(transaction: object, scope?: IssueReco } } +export function issueRecoveryBackstopHint(companyId: string) { + return JSON.stringify({ companyId, backstop: true }); +} + export function decodeIssueRecoveryHint(hint: string): IssueRecoveryScope | undefined { return hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope; } @@ -44,10 +48,7 @@ export async function notifyIssueGraphRecovery(transaction: object, const companyHint = JSON.stringify({ companyId: scope.companyId }); for (const queue of COMPANY_QUEUES) await notifyDeliveryWork(transaction, queue, companyHint); const graphHint = JSON.stringify({ companyId: scope.companyId, graphRunId: scope.runId ?? undefined, graphIssueId: scope.issueId ?? undefined }); - for (const queue of TASK_QUEUES) { - await notifyDeliveryWork(transaction, queue, graphHint); - await notifyDeliveryWork(transaction, queue, JSON.stringify({ companyId: scope.companyId, backstop: true })); - } + for (const queue of TASK_QUEUES) await notifyDeliveryWork(transaction, queue, graphHint); } export async function resolveIssueGraph(db: Db, scope: { companyId: string; runId?: string; issueId?: string }): Promise { diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index cd12fdcd29c..7a855a721d0 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -127,7 +127,7 @@ import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; import { budgetService } from "../budgets.js"; import { isBoardOwnedBudgetBlock } from "../budget-block-ownership.js"; import { unadmittedChatWakeupCondition } from "../durable-chat-wakeup.js"; -import { issueRecoveryActionService } from "../issue-recovery-actions.js"; +import { issueRecoveryActionService, SOURCE_TERMINAL_RESOLUTION } from "../issue-recovery-actions.js"; import { PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS } from "../system-recovery-state.js"; import { parseProviderQuotaClockReset } from "../provider-quota-reset.js"; import { hasUnverifiedNativeWorkspaceFinalizationOwner } from "../native-runtime/native-workspace-finalization-recovery-state.js"; @@ -3670,7 +3670,8 @@ export function recoveryService( actionId: action.id, status: "resolved", outcome: "restored", - resolutionNote: "source_terminal", + resolutionNote: SOURCE_TERMINAL_RESOLUTION, + sourceTerminal: true, }); if (resolved) { result.resolved += 1; diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index 7c9e02c2271..6fae89d60e0 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -3,7 +3,7 @@ import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql, type SQL } from import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js"; -import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, issueRecoveryScopeMatches, resolveIssueGraph, type IssueRecoveryScope } +import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, issueRecoveryBackstopHint, issueRecoveryScopeMatches, resolveIssueGraph, type IssueRecoveryScope } from "../issue-recovery-notifications.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; import { workSignalDebug } from "@paperclipai/db/work-signal-debug"; @@ -71,6 +71,7 @@ export function registerIssueRecoveryWork( scope.wholeCompany = ids === null; } async function toCompany(scope: IssueRecoveryScope | undefined, worker: { invalidate(hint?: string): void }) { + if (scope && isGraph(scope)) worker.invalidate(issueRecoveryBackstopHint(scope.companyId)); await resolveGraph(scope); if (!scope?.wholeCompany) return false; worker.invalidate(JSON.stringify({ companyId: scope.companyId }));