diff --git a/server/src/__tests__/issue-recovery-recorded-decision-idle.test.ts b/server/src/__tests__/issue-recovery-recorded-decision-idle.test.ts new file mode 100644 index 00000000000..3cbbd7d2504 --- /dev/null +++ b/server/src/__tests__/issue-recovery-recorded-decision-idle.test.ts @@ -0,0 +1,88 @@ +import { setImmediate as immediate } from "node:timers/promises"; +import { and, eq, sql } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { agents, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; + +describeIssueRecovery("stranded recovery of runs with a recorded decision", () => { + const state = issueRecoveryFixture(); + const TASKS = 10; + + async function seed() { + const f = await state.fixture(); + const db = state.db; + await db.update(agents).set({ status: "error" }).where(eq(agents.id, f.agentId)); + const tasks = await db.insert(issues).values(Array.from({ length: TASKS }, (_, index) => ({ + companyId: f.companyId, title: `Waiting task ${index}`, status: "in_progress", assigneeAgentId: f.agentId, + }))).returning({ id: issues.id }); + await db.insert(heartbeatRuns).values(tasks.map(task => ({ companyId: f.companyId, agentId: f.agentId, + status: "failed", errorCode: "adapter_failed", finishedAt: new Date(Date.now() - 60 * 60_000), + contextSnapshot: { issueId: task.id } }))); + return { ...f, tasks }; + } + + async function advance(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { + await vi.advanceTimersByTimeAsync(Math.min(500, until - Date.now())); + await Promise.allSettled([...state.passes]); await immediate(); + } + } + + async function countStatements(work: () => Promise) { + const statements: string[] = [], prior = state.db.$client.options.debug; + state.db.$client.options.debug = (_connection, query) => { statements.push(query); }; + try { await work(); } finally { state.db.$client.options.debug = prior; } + return statements; + } + + async function startAndRecordDecisions(f: Awaited>) { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await state.start({}); + await advance(6 * 60_000); + const decided = await state.db.select({ issueId: issueRecoveryActions.sourceIssueId }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, f.companyId), eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence} ? 'executionReconciliation'`)); + expect(new Set(decided.map(row => row.issueId))).toEqual(new Set(f.tasks.map(task => task.id))); + } + + it("stays quiet for half an hour once each failed run has its decision", async () => { + const f = await seed(); + await startAndRecordDecisions(f); + const runsBefore = await state.db.select({ id: heartbeatRuns.id, status: heartbeatRuns.status, updatedAt: heartbeatRuns.updatedAt }) + .from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + const idle = await countStatements(() => advance(30 * 60_000)); + expect(idle).toHaveLength(0); + const runsAfter = await state.db.select({ id: heartbeatRuns.id, status: heartbeatRuns.status, updatedAt: heartbeatRuns.updatedAt }) + .from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + const byId = (a: { id: string }, b: { id: string }) => a.id.localeCompare(b.id); + expect(runsAfter.sort(byId)).toEqual(runsBefore.sort(byId)); + const statuses = await state.db.select({ status: issues.status }).from(issues).where(eq(issues.companyId, f.companyId)); + expect(statuses.every(row => row.status === "in_progress")).toBe(true); + }, 300_000); + + it("still recovers promptly when another run of a waiting task fails", async () => { + const f = await seed(); + await startAndRecordDecisions(f); + await advance(10 * 60_000); + const target = f.tasks[0].id; + const [failed] = await state.db.transaction(async tx => { + const rows = await tx.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "failed", + errorCode: "adapter_failed", finishedAt: new Date(), contextSnapshot: { issueId: target } }).returning(); + await notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: rows[0].id }); + return rows; + }); + const actionFor = async () => (await state.db.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, f.companyId), eq(issueRecoveryActions.sourceIssueId, target), + eq(issueRecoveryActions.fingerprint, `legacy-execution:${failed.id}`))))[0]; + await advance(5_000); + expect(await actionFor()).toMatchObject({ kind: "active_run_watchdog", evidence: expect.objectContaining({ runId: failed.id }) }); + await advance(6 * 60_000); + expect(await actionFor()).toMatchObject({ status: "resolved", + evidence: expect.objectContaining({ executionReconciliation: expect.objectContaining({ runId: failed.id }) }) }); + const after = await countStatements(() => advance(30 * 60_000)); + expect(after.filter(query => query.startsWith("update \"heartbeat_runs\""))).toEqual([]); + expect(after.length).toBeLessThan(TASKS * 500); + }, 300_000); +}); diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 6d48112de2d..0f9fe34c63d 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -111,6 +111,58 @@ export async function legacyExecutionNeedsReconciliationWithEvidence(db: Db, run && !(await isPreDispatchExecutionWaitVerified(db, run))); } +async function findRecordedLegacyDecision(db: Db, run: Run, taskId: string, resultJson: Run["resultJson"]) { + const [reconciled] = await db.select({ id: issueRecoveryActions.id }) + .from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, run.companyId), + eq(issueRecoveryActions.sourceIssueId, taskId), + eq(issueRecoveryActions.status, "resolved"), + ...(hasRequiredWorkspaceRecovery(resultJson) ? [ + sql`${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`, + ] : []), + or( + sql`${issueRecoveryActions.evidence}->'executionReconciliation'->>'runId' = ${run.id}`, + and( + sql`${issueRecoveryActions.evidence}->>'runId' = ${run.id}`, + sql`exists ( + select 1 from heartbeat_runs successor + where successor.id = ${canonicalExecutionUuid(sql`${issueRecoveryActions.evidence}->'explicitUserContinuation'->>'runId'`)} + and successor.company_id = ${run.companyId} and successor.agent_id = ${run.agentId} + and coalesce(successor.native_issue_id::text, successor.issue_id::text, + successor.context_snapshot->>'issueId') = ${taskId} + and successor.context_snapshot->>'previousRunId' = ${issueRecoveryActions.evidence}->'explicitUserContinuation'->>'previousRunId' + and successor.context_snapshot->'explicitUserContinuation'->>'previousRunId' = ${issueRecoveryActions.evidence}->'explicitUserContinuation'->>'previousRunId' + and successor.context_snapshot->'explicitUserContinuation'->>'commentId' is not distinct from + ${issueRecoveryActions.evidence}->'explicitUserContinuation'->>'commentId' + )`, + ), + and( + sql`${issueRecoveryActions.evidence}->>'runId' = ${run.id}`, + or(sql`${issueRecoveryActions.evidence}->>'workspaceRestoreFailure' = 'restore_unsafe_archive'`, + sql`${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`), + sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, + ), + and( + eq(issueRecoveryActions.cause, LEGACY_RECOVERY_CAUSE), + eq(issueRecoveryActions.fingerprint, `legacy-execution:${run.id}`), + sql`${issueRecoveryActions.evidence}->>'runId' = ${run.id}`, + sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, + ), + ), + )).limit(1); + return reconciled ?? null; +} + +function decisionOwner(task: typeof issues.$inferSelect, run: Run, retainedLeaseCount: number) { + const review = task.status === "in_review" ? parseIssueExecutionState(task.executionState) : null; + const isCurrentReviewer = review?.status === "pending" && + review.currentParticipant?.type === "agent" && review.currentParticipant.agentId === run.agentId; + const owns = !isSupersededConversationRun(task, run) && + (task.assigneeAgentId === run.agentId || isCurrentReviewer) && + (!["done", "cancelled"].includes(task.status) || retainedLeaseCount > 0); + return { owns, isCurrentReviewer }; +} + /** Persist the failed legacy run, owned lock release and operator decision together. */ export async function terminalizeLegacyExecution(input: { db: Db; @@ -152,6 +204,16 @@ export async function terminalizeLegacyExecution(input: { )).for("update"); if (!current || (current.runtimeMode !== "legacy" && !(input.recordRestoreFailureOnly && current.runtimeMode === "native")) || (!input.recordRestoreFailureOnly && !(input.fromStatuses ?? [run.status]).includes(current.status))) return null; + if (!input.recordRestoreFailureOnly && current.status === status && Object.keys(patch ?? {}).length === 0 + && !hasWorkspaceRestoreFailure(current.resultJson) && !hasRequiredWorkspaceRecovery(current.resultJson) + && task?.executionRunId !== run.id && task?.checkoutRunId !== run.id) { + const [unchanged] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId))); + if (input.reconcileIfNeeded && !(await legacyExecutionNeedsReconciliationWithEvidence(tx as unknown as Db, unchanged))) + return unchanged; + if (!task || !decisionOwner(task, unchanged, 0).owns) return unchanged; + if (await findRecordedLegacyDecision(tx as unknown as Db, run, task.id, unchanged.resultJson)) return unchanged; + } if (patch?.resultJson !== undefined) patch = { ...patch, resultJson: preserveWorkspaceRestoreRecoveryMetadata(current.resultJson, input.recordRestoreFailureOnly ? { ...current.resultJson, ...patch.resultJson } : patch.resultJson), @@ -255,58 +317,15 @@ export async function terminalizeLegacyExecution(input: { fingerprint, attemptCount: 1 }); return updated; } - const review = task?.status === "in_review" ? parseIssueExecutionState(task.executionState) : null; - const isCurrentReviewer = review?.status === "pending" && - review.currentParticipant?.type === "agent" && review.currentParticipant.agentId === run.agentId; - if ( - task && - !isSupersededConversationRun(task, updated) && - (task.assigneeAgentId === run.agentId || isCurrentReviewer) && - (!["done", "cancelled"].includes(task.status) || retainedLeaseIds.length > 0) - ) { + const owner = task ? decisionOwner(task, updated, retainedLeaseIds.length) : null; + if (task && owner?.owns) { + const { isCurrentReviewer } = owner; // Periodic stranded-work checks may revisit this terminal run before its // reconciled or explicitly requested continuation is dispatched. Preserve // its recorded decision instead of creating another hold. The explicit // receipt must identify an actual successor under the same task/agent; // retained user intent remains authoritative after that successor finishes. - const [reconciled] = await tx.select({ id: issueRecoveryActions.id }) - .from(issueRecoveryActions).where(and( - eq(issueRecoveryActions.companyId, run.companyId), - eq(issueRecoveryActions.sourceIssueId, task.id), - eq(issueRecoveryActions.status, "resolved"), - ...(hasRequiredWorkspaceRecovery(updated.resultJson) ? [ - sql`${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`, - ] : []), - or( - sql`${issueRecoveryActions.evidence}->'executionReconciliation'->>'runId' = ${run.id}`, - and( - sql`${issueRecoveryActions.evidence}->>'runId' = ${run.id}`, - sql`exists ( - select 1 from heartbeat_runs successor - where successor.id = ${canonicalExecutionUuid(sql`${issueRecoveryActions.evidence}->'explicitUserContinuation'->>'runId'`)} - and successor.company_id = ${run.companyId} and successor.agent_id = ${run.agentId} - and coalesce(successor.native_issue_id::text, successor.issue_id::text, - successor.context_snapshot->>'issueId') = ${task.id} - and successor.context_snapshot->>'previousRunId' = ${issueRecoveryActions.evidence}->'explicitUserContinuation'->>'previousRunId' - and successor.context_snapshot->'explicitUserContinuation'->>'previousRunId' = ${issueRecoveryActions.evidence}->'explicitUserContinuation'->>'previousRunId' - and successor.context_snapshot->'explicitUserContinuation'->>'commentId' is not distinct from - ${issueRecoveryActions.evidence}->'explicitUserContinuation'->>'commentId' - )`, - ), - and( - sql`${issueRecoveryActions.evidence}->>'runId' = ${run.id}`, - or(sql`${issueRecoveryActions.evidence}->>'workspaceRestoreFailure' = 'restore_unsafe_archive'`, - sql`${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`), - sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, - ), - and( - eq(issueRecoveryActions.cause, LEGACY_RECOVERY_CAUSE), - eq(issueRecoveryActions.fingerprint, `legacy-execution:${run.id}`), - sql`${issueRecoveryActions.evidence}->>'runId' = ${run.id}`, - sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, - ), - ), - )).limit(1); + const reconciled = await findRecordedLegacyDecision(tx as unknown as Db, run, task.id, updated.resultJson); if (reconciled) return updated; await issueRecoveryActionService(tx as unknown as Db).upsertSourceScoped({ companyId: run.companyId,