diff --git a/server/src/__tests__/task-interruption-automatic-path.test.ts b/server/src/__tests__/task-interruption-automatic-path.test.ts index 17443fd2171..74f0da150a3 100644 --- a/server/src/__tests__/task-interruption-automatic-path.test.ts +++ b/server/src/__tests__/task-interruption-automatic-path.test.ts @@ -4,6 +4,7 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, agentWakeupRequests, applyPendingMigrations, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js"; +import { createSystemRecovery } from "../services/system-recovery-state.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; @@ -96,6 +97,106 @@ describe.skipIf(!support.supported)("Retry is offered only when no automatic pat expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ canRetry: true }); }); + async function seedParentWaitingOnSubtasks(childStatus = "in_progress") { + const { companyId, agentId, issue } = await seedTask("blocked"); + for (const title of ["Subtask 1", "Subtask 2"]) + await db.insert(issues).values({ companyId, title, status: childStatus, assigneeAgentId: agentId, parentId: issue.id }); + const runId = randomUUID(), finishedAt = new Date(Date.now() - 1_000); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issue.id, scopeKind: "issue", + status: "succeeded", runtimeMode: "native", startedAt: new Date(Date.now() - 25 * 60_000), finishedAt }); + return { companyId, agentId, issue, runId }; + } + + async function seedCommentQueuedDuringRun(companyId: string, agentId: string, issueId: string) { + const commentId = randomUUID(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "automation", + reason: "issue_execution_deferred", triggerDetail: "system", status: "deferred_issue_execution", + requestedByActorType: "user", requestedByActorId: "local-board", requestedAt: new Date(Date.now() - 13 * 60_000), + payload: { issueId, mutation: "comment", commentId, + _paperclipWakeContext: { source: "issue.comment", issueId, wakeReason: "issue_commented", wakeCommentIds: [commentId] } }, + }).returning(); + return wake!; + } + + it("shows a parent whose run just succeeded waiting on its subtasks as working while its queued message is settled", async () => { + const { companyId, agentId, issue, runId } = await seedParentWaitingOnSubtasks(); + await seedCommentQueuedDuringRun(companyId, agentId, issue.id); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ responsibility: "system", state: "working", + canRetry: false, observedRunId: runId }); + }); + + it("shows the parent as resolved once its queued message is cancelled as already delivered", async () => { + const { companyId, agentId, issue } = await seedParentWaitingOnSubtasks(); + const wake = await seedCommentQueuedDuringRun(companyId, agentId, issue.id); + await db.update(agentWakeupRequests).set({ status: "cancelled", finishedAt: new Date(), + error: "Queued messages were already delivered to a completed run" }).where(eq(agentWakeupRequests.id, wake.id)); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: "resolved", canRetry: false }); + }); + + it("shows the parent as working once its queued message is promoted to a new run", async () => { + const { companyId, agentId, issue } = await seedParentWaitingOnSubtasks(); + const wake = await seedCommentQueuedDuringRun(companyId, agentId, issue.id); + const nextRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId, nativeIssueId: issue.id, scopeKind: "issue", + status: "queued", runtimeMode: "native", wakeupRequestId: wake.id }); + await db.update(agentWakeupRequests).set({ status: "queued", runId: nextRunId }).where(eq(agentWakeupRequests.id, wake.id)); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: "working", canRetry: false, observedRunId: nextRunId }); + }); + + it("offers Retry on the parent once its queued message is not settled within the grace", async () => { + const { companyId, agentId, issue, runId } = await seedParentWaitingOnSubtasks(); + await seedCommentQueuedDuringRun(companyId, agentId, issue.id); + await db.update(heartbeatRuns).set({ finishedAt: new Date(Date.now() - 3 * 60_000) }).where(eq(heartbeatRuns.id, runId)); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: "failed", canRetry: true }); + }); + + it("still offers Retry on a blocked task with a queued message and no open subtasks", async () => { + for (const childStatus of [null, "done", "cancelled"]) { + const { companyId, agentId, issue } = await seedTask("blocked"); + if (childStatus) await db.insert(issues).values({ companyId, title: "Finished subtask", status: childStatus, + assigneeAgentId: agentId, parentId: issue.id }); + await db.insert(heartbeatRuns).values({ companyId, agentId, nativeIssueId: issue.id, scopeKind: "issue", + status: "succeeded", runtimeMode: "native", finishedAt: new Date(Date.now() - 1_000) }); + await seedCommentQueuedDuringRun(companyId, agentId, issue.id); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: "failed", canRetry: true }); + } + }); + + it("still offers Retry on a parent with open subtasks when its queued wake carries no message", async () => { + const { companyId, agentId, issue } = await seedParentWaitingOnSubtasks(); + await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "on_demand", reason: "issue_assigned", + status: "deferred_issue_execution", requestedByActorType: "user", requestedAt: new Date(Date.now() - 13 * 60_000), + payload: { issueId: issue.id } }); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: "failed", canRetry: true }); + }); + + it("still offers Retry on a parent with open subtasks and a queued message when its run failed and recovery gave up", async () => { + const { companyId, agentId, issue, runId, hold } = await seedSwapEndedRun(new Date(Date.now() - 5_000)); + await db.update(issues).set({ status: "blocked" }).where(eq(issues.id, issue.id)); + await db.insert(issues).values({ companyId, title: "Open subtask", status: "in_progress", assigneeAgentId: agentId, parentId: issue.id }); + await seedCommentQueuedDuringRun(companyId, agentId, issue.id); + await db.update(issueRecoveryActions).set({ ownerType: "board" }).where(eq(issueRecoveryActions.id, hold.id)); + const blocked = { ...issue, status: "blocked" }; + expect(await taskInterruptionPresentation(db, blocked)).toMatchObject({ state: "failed", canRetry: true, observedRunId: runId }); + }); + + it("still offers Retry on a parent with open subtasks and a queued message when saved-work recovery is exhausted", async () => { + const { companyId, agentId, issue, runId } = await seedParentWaitingOnSubtasks(); + await seedCommentQueuedDuringRun(companyId, agentId, issue.id); + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issue.id, kind: "active_run_watchdog", + status: "escalated", ownerType: "system", returnOwnerAgentId: agentId, cause: "native_restart_recovery_blocked", + fingerprint: `exhausted:${runId}`, nextAction: "Retry to continue.", maxAttempts: 3, + evidence: { runId, systemRecovery: { ...createSystemRecovery(runId), state: "exhausted", attempts: 3, retriesUsed: 3 } } }); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: "failed", canRetry: true }); + }); + + it("does not show a failed attempt on a parent whose run succeeded with no queued message", async () => { + for (const childStatus of ["in_progress", "blocked", "cancelled"]) { + const { issue } = await seedParentWaitingOnSubtasks(childStatus); + expect(await taskInterruptionPresentation(db, issue)).toMatchObject({ state: "resolved", canRetry: false }); + } + }); + it("does not offer Retry on a parent whose run succeeded and that waits for its subtasks", async () => { const { companyId, agentId, issue } = await seedTask(); await db.insert(issues).values({ companyId, title: "Subtask", status: "in_progress", assigneeAgentId: agentId, parentId: issue.id }); diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 01a72a947c4..d0b9c1dd35f 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -1,7 +1,7 @@ import { STORAGE_UNAVAILABLE_WAIT_RETRY_REASON } from "../storage/storage-unavailable.js"; import { AGENT_DIRECTORY_WAIT_RETRY_REASON } from "./execution-recovery-attempt.js"; import { boundSetupCommandFailure, readSetupCommandFailure } from "./workspace-setup-command-failure.js"; -import { and, desc, eq, inArray, isNull, not, sql, type SQL } from "drizzle-orm"; +import { and, desc, eq, inArray, isNull, not, notInArray, sql, type SQL } from "drizzle-orm"; import { unidentifiedLocalLaunchRefusals } from "./native-local-process-stop.js"; import { nativeSessionCleanupBlock } from "../vendor/paperclip-runner/index.js"; import { agents, agentWakeupRequests, approvals, chatActions, companies, environmentLeases, heartbeatRuns, issueApprovals, issueRecoveryActions, @@ -65,7 +65,8 @@ type ProjectionRun = Pick now - 10_000 || queuedDuringRun) && system?.state !== "exhausted") return { ...base, state: "working" }; @@ -496,7 +499,10 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i executionWaitReason: boundedJsonString(sql`to_jsonb(${unpricedUsageWait(sql`${agentWakeupRequests.payload}#>>'{executionWait,reason}'`, sql`${agentWakeupRequests.payload}#>>'{executionWait,message}'`)})`, 80), preparationStatus: chatActions.status, preparationRetryAt: chatActionDeadline("failed_run_retry"), - preparationUpdatedAt: chatActions.updatedAt }) + preparationUpdatedAt: chatActions.updatedAt, + queuedComment: sql`coalesce(jsonb_typeof(${agentWakeupRequests.payload}->'commentId') = 'string' + or jsonb_array_length(case when jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array' + then ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' end) > 0, false)` }) .from(agentWakeupRequests).leftJoin(chatActions, and(eq(chatActions.companyId, agentWakeupRequests.companyId), eq(chatActions.id, canonicalExecutionUuid(sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}'`)), eq(chatActions.kind, "failed_run_retry"), sql`${chatActions.payload}->>'taskRetryIntentId' = ${agentWakeupRequests.id}::text`, @@ -670,6 +676,11 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i for (const [waitingIssueId, row] of readiness) if (row.unresolvedBlockerIssueIds.length) waitingOnBlockers.add(waitingIssueId); } + const parentIds = selected.filter(issue => issue.status === "blocked" && runMap.get(issue.id)?.status === "succeeded" + && currentWaits.some(wait => wait.issueId === issue.id && wait.queuedComment)).map(issue => issue.id); + const waitingOnOpenSubtasks = new Set(parentIds.length ? (await db.selectDistinct({ parentId: issues.parentId }).from(issues) + .where(and(eq(issues.companyId, companyId), inArray(issues.parentId, parentIds), notInArray(issues.status, ["done", "cancelled"])))) + .flatMap(row => row.parentId ? [row.parentId] : []) : []); for (const wait of currentWaits) { const current = waitMap.get(wait.issueId); if (!current || wait.requestedAt > current.requestedAt || @@ -736,6 +747,7 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i admissionWait: admissionWaitMap.get(issue.id), retryDependencies: retryDependencyMap.get(issue.id), waitingOnBlockers: waitingOnBlockers.has(issue.id), + waitingOnOpenSubtasks: waitingOnOpenSubtasks.has(issue.id), cleanupSettling: cleanupSettling.has(issue.id), }, now); result.set(issue.id, presentation);