diff --git a/server/src/__tests__/task-interruption-automatic-path.test.ts b/server/src/__tests__/task-interruption-automatic-path.test.ts index 3c9c179760d..1fea7c136d6 100644 --- a/server/src/__tests__/task-interruption-automatic-path.test.ts +++ b/server/src/__tests__/task-interruption-automatic-path.test.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; +import { and, eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, agentWakeupRequests, applyPendingMigrations, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; @@ -158,6 +158,70 @@ describe.skipIf(!support.supported)("Retry is offered only when no automatic pat expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: "failed", canRetry: true }); }); + async function seedProviderTerminalFailure(finishedAt: Date) { + const task = await seedTask("blocked"); + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: task.companyId, agentId: task.agentId, + nativeIssueId: task.issue.id, scopeKind: "issue", status: "failed", runtimeMode: "native", + nativePhase: "terminal_failure", errorCode: "native_provider_terminal_failed", + error: "ACP agent reported a terminal service failure.", startedAt: new Date(finishedAt.getTime() - 60_000), finishedAt }); + await db.insert(nativeRunFinalizations).values({ companyId: task.companyId, issueId: task.issue.id, runId, + phase: "terminal_failure", attempt: 1, failureCode: "native_provider_terminal_failed", updatedAt: finishedAt }); + const [hold] = await db.insert(issueRecoveryActions).values({ companyId: task.companyId, sourceIssueId: task.issue.id, + kind: "active_run_watchdog", status: "active", ownerType: "board", returnOwnerAgentId: task.agentId, + cause: "native_provider_terminal_failed", fingerprint: `provider-terminal:${runId}`, + evidence: { runId, nativeFailureBlock: { runId, statusVersion: 1 }, coordinatorAttempt: 1, + sourceFailureCode: "native_provider_terminal_failed", recoveryDisposition: "native_provider_terminal_failed", + recoveryMode: "replacement", providerSessionEstablished: true }, + nextAction: "Verify that the failed provider stopped and reconcile its action outcomes.", + maxAttempts: 3, createdAt: finishedAt, updatedAt: finishedAt }).returning(); + return { ...task, issue: { ...task.issue, status: "blocked" }, runId, hold: hold! }; + } + + it("shows a provider failure the system is still recovering as working through every recovery step, never offering Retry in between", async () => { + const { issue, runId, hold, companyId } = await seedProviderTerminalFailure(new Date(Date.now() - 2_000)); + const working = { responsibility: "system", state: "working", canRetry: false, observedRunId: runId }; + expect(await taskInterruptionPresentation(db, issue)).toMatchObject(working); + + const running = { ...createSystemRecovery(runId), state: "running" as const, phase: "preserve_files" as const }; + await db.update(issueRecoveryActions).set({ ownerType: "system", updatedAt: new Date(), + evidence: { ...hold.evidence, systemRecovery: running, systemRecoveryActiveAt: Date.now() } }) + .where(eq(issueRecoveryActions.id, hold.id)); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject(working); + + await db.update(issueRecoveryActions).set({ updatedAt: new Date(), + evidence: { ...hold.evidence, systemRecovery: { ...running, state: "complete" }, systemRecoveryActiveAt: Date.now() - 60_000 } }) + .where(eq(issueRecoveryActions.id, hold.id)); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject(working); + + await db.update(nativeRunFinalizations).set({ failureDetail: { replacementDenied: "provider_failure_meaning_unverified" }, + updatedAt: new Date() }).where(eq(nativeRunFinalizations.runId, runId)); + await db.update(issueRecoveryActions).set({ ownerType: "board", cause: "provider_failure_meaning_unverified", + fingerprint: `native-replacement:${runId}`, evidence: { runId, attempts: 1 }, nextAction: "Review the provider failure.", + updatedAt: new Date() }).where(and(eq(issueRecoveryActions.id, hold.id), eq(issueRecoveryActions.companyId, companyId))); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject(working); + }); + + it("offers Retry for a provider failure once the recovery bound passes with nothing recorded", async () => { + const { issue } = await seedProviderTerminalFailure(new Date(Date.now() - 3 * 60_000)); + expect(await taskInterruptionPresentation(db, { ...issue, status: "todo" })).toMatchObject({ state: "failed", canRetry: true }); + }); + + it("offers Retry for a provider failure at once when the continuation worker decided it cannot continue the task", async () => { + const { issue, hold } = await seedProviderTerminalFailure(new Date(Date.now() - 2_000)); + await db.update(issueRecoveryActions).set({ evidence: { ...hold.evidence, + freshContinuationCheck: { kind: "not_applicable", reason: "task_changed", checks: 1, checkedAt: Date.now(), nextCheckAt: null } } }) + .where(eq(issueRecoveryActions.id, hold.id)); + expect(await taskInterruptionPresentation(db, { ...issue, status: "todo" })).toMatchObject({ state: "failed", canRetry: true }); + }); + + it("offers Retry for a provider failure at once when a person took over its hold", async () => { + const { issue, hold, companyId } = await seedProviderTerminalFailure(new Date(Date.now() - 2_000)); + await db.update(issueRecoveryActions).set({ evidence: { ...hold.evidence, ownerGoneAttestation: { companyId } } }) + .where(eq(issueRecoveryActions.id, hold.id)); + expect(await taskInterruptionPresentation(db, { ...issue, status: "todo" })).toMatchObject({ state: "failed", canRetry: true }); + }); + it("shows a healthy task as working while the message queued during its succeeded run is delivered", async () => { const { companyId, agentId, issue } = await seedTask(); const runId = randomUUID(), finishedAt = new Date(Date.now() - 3_000); diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 2f3d1dc02bd..53020da5e06 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -52,6 +52,8 @@ type ProjectionRun = Pick`(select ${nativeRunFinalizations.updatedAt} from ${nativeRunFinalizations} where ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId} and ${nativeRunFinalizations.runId} = ${heartbeatRuns.id} and ${nativeRunFinalizations.phase} = 'terminal_failure')`.mapWith(value => value == null ? null : new Date(value)), + providerRecoveryAt: sql`(select ${nativeRunFinalizations.updatedAt} from ${nativeRunFinalizations} + where ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId} and ${nativeRunFinalizations.runId} = ${heartbeatRuns.id} + and ${heartbeatRuns.runtimeMode} = 'native' and ${heartbeatRuns.status} = 'failed' + and ${nativeRunFinalizations.phase} = 'terminal_failure' and ${nativeRunFinalizations.failureCode} = 'native_provider_terminal_failed' + and ${nativeRunFinalizations.resultId} is null and ${nativeRunFinalizations.leaseOwner} is null + and coalesce(${nativeRunFinalizations.failureDetail}->>'successorRunId', '') = '')`.mapWith(value => value == null ? null : new Date(value)), nativeFinalizationRetry: exhaustedNativeFinalizationCondition(), nativeFinalizationRetryPending: savedResultRetryPendingCondition(), // The run whose resultJson supplied providerFailure. A scheduled retry @@ -613,6 +626,13 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]), isNull(issueRecoveryActions.ownerUserId), sql`not (${issueRecoveryActions.evidence} ?| array['systemRecovery', 'executionReconciliation', 'freshContinuation', 'freshContinuationCheck', 'ownerGoneAttestation', 'explicitUserContinuation'])`)) : []; + const recoveringRunIds = runs.flatMap(run => run.providerRecoveryAt ? [run.id] : []); + const handedToPerson = new Set((recoveringRunIds.length ? await db.select({ issueId: issueRecoveryActions.sourceIssueId }) + .from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, ids), + inArray(sql`${issueRecoveryActions.evidence}->>'runId'`, recoveringRunIds), + sql`(${issueRecoveryActions.evidence} ?| array['ownerGoneAttestation', 'explicitUserContinuation'] + or coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,kind}', 'exact_recovery_pending') not in ('exact_recovery_pending', 'backoff'))`)) : []) + .map(row => row.issueId)); const automaticDispositionAt = new Map(); if (dispositionRows.length) { const { isPersonOwnedFailureCode } = await import("./fresh-continuation-fallback.js"); @@ -735,6 +755,7 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i const presentation = projectTaskInterruption(issue, { ...facts, freshContinuation: freshContinuationFor(issue.id, facts.run?.id), automaticDispositionAt: automaticDispositionAt.get(issue.id), + recoveryHandedToPerson: handedToPerson.has(issue.id), interruptWait: interruptWaitFor(issue.id), projectAccessAllowed: denied?.length === 0, projectAccessProjectId: denied?.includes(issue.projectId ?? "") ? issue.projectId : null,