diff --git a/server/src/__tests__/assigned-work-invariant.test.ts b/server/src/__tests__/assigned-work-invariant.test.ts index ca477a532d7..18b449ee20f 100644 --- a/server/src/__tests__/assigned-work-invariant.test.ts +++ b/server/src/__tests__/assigned-work-invariant.test.ts @@ -182,10 +182,83 @@ describePostgres("assigned-work invariant sweep", () => { expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status).toBe("in_progress"); }, 60_000); + it("a recovery block whose task already has a queued run is lifted without a second run", async () => { + const f = await seedCompany(); + const blockedAt = new Date(Date.now() - 20 * 60_000); + const issueId = await f.issue({ status: "blocked", blockedTransitionAt: blockedAt }); + await f.run(issueId, { runtimeMode: "native", status: "failed", errorCode: "native_continuation_requires_reconciliation", + livenessState: "failed", finishedAt: new Date(blockedAt.getTime() - 5_000) }); + await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "execution-recovery", action: "issue.updated", + entityType: "issue", entityId: issueId, details: { status: "blocked", previousStatus: "in_progress", + reason: "native_continuation_requires_reconciliation" }, createdAt: blockedAt }); + const successor = await f.run(issueId, { status: "queued", errorCode: null, livenessState: null, startedAt: null, finishedAt: null, + createdAt: new Date(blockedAt.getTime() + 19_000) }); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", + triggerDetail: "system", reason: "issue_continuation_needed", status: "queued", runId: successor.id, + payload: { issueId } }).returning(); + await db.update(heartbeatRuns).set({ wakeupRequestId: wake.id }).where(eq(heartbeatRuns.id, successor.id)); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup }); + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id)).toContain(issueId); + expect(await recovery.assignedWork.report(f.companyId)).toMatchObject({ violations: { blocked_with_live_run: 1 } }); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })) + .toMatchObject({ outcome: "repaired", verdict: { reason: "blocked_with_live_run", runId: successor.id } }); + expect(queue.calls).toHaveLength(0); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status).toBe("todo"); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })) + .toMatchObject({ outcome: "ok", verdict: { clause: "live_run" } }); + }, 60_000); + + it("continues a recent board escalation of a temporary setup failure itself, and leaves old or lasting ones", async () => { + const f = await seedCompany(); + async function escalated(input: { endedAgoMs: number; cause: string }) { + const blockedAt = new Date(Date.now() - input.endedAgoMs + 1_000); + const issueId = await f.issue({ status: "blocked", blockedTransitionAt: blockedAt }); + const run = await f.run(issueId, { status: "failed", errorCode: "reusable_sandbox_resume_failed", livenessState: "failed", + error: `Failed to acquire lease: Reusable sandbox lease "x" could not be resumed (cause=${input.cause}); the lease was preserved and no replacement was created.`, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + startedAt: new Date(Date.now() - input.endedAgoMs - 60_000), finishedAt: new Date(Date.now() - input.endedAgoMs), + createdAt: new Date(Date.now() - input.endedAgoMs - 60_000) }); + const [hold] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: issueId, + kind: "stranded_assigned_issue", status: "active", ownerType: "board", cause: "stranded_assigned_issue", + fingerprint: `source_scoped_recovery:${f.companyId}:${issueId}:stranded_assigned_issue`, + evidence: { previousStatus: "in_progress", latestRunId: run.id, latestRunErrorCode: "reusable_sandbox_resume_failed", + routingPolicy: "board_escalation_no_takeover_v1" }, + nextAction: "Board operator: inspect the evidence, repair the runtime if appropriate.", attemptCount: 1 }).returning(); + await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "system", action: "issue.updated", + entityType: "issue", entityId: issueId, details: { status: "blocked", previousStatus: "in_progress", + source: "recovery.reconcile_stranded_assigned_issue" }, createdAt: blockedAt }); + return { issueId, hold }; + } + const recent = await escalated({ endedAgoMs: 2 * HOUR, cause: "transport_timeout" }); + const old = await escalated({ endedAgoMs: 30 * HOUR, cause: "transport_timeout" }); + const lasting = await escalated({ endedAgoMs: 2 * HOUR, cause: "provider_access_denied" }); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup }); + const suspects = (await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id); + expect(suspects).toContain(recent.issueId); + expect(suspects).not.toContain(old.issueId); + + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: recent.issueId })) + .toMatchObject({ outcome: "repaired", verdict: { reason: "recovery_block" } }); + expect(queue.calls).toEqual([{ agentId: f.agentId, issueId: recent.issueId, source: "issue.assigned_work_invariant" }]); + expect((await db.select().from(issues).where(eq(issues.id, recent.issueId)))[0].status).toBe("in_progress"); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, recent.hold.id)))[0]) + .toMatchObject({ status: "resolved", outcome: "restored", ownerType: "system", resolutionNote: "stranded_transient_continued" }); + + for (const left of [old, lasting]) { + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: left.issueId })) + .toMatchObject({ outcome: "left_to_hold_owner" }); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, left.hold.id)))[0].status).toBe("active"); + } + expect(queue.calls).toHaveLength(1); + }, 60_000); + it("leaves a task blocked under an active recovery hold to the hold's owner and reports it", async () => { const f = await seedCompany(); const issueId = await f.issue({ status: "blocked", blockedTransitionAt: new Date(Date.now() - 2 * HOUR) }); - const run = await f.run(issueId, { status: "failed", errorCode: "process_lost", livenessState: "failed" }); + const run = await f.run(issueId, { status: "failed", errorCode: "process_lost", livenessState: "failed", + startedAt: new Date(Date.now() - 31 * HOUR), finishedAt: new Date(Date.now() - 30 * HOUR), createdAt: new Date(Date.now() - 31 * HOUR) }); // Old-shaped row: a board-owned stranded action written by an older release. await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: issueId, kind: "stranded_assigned_issue", status: "active", ownerType: "board", cause: "stranded_assigned_issue", fingerprint: `source_scoped_recovery:${f.companyId}:${issueId}:stranded_assigned_issue`, diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 68060a5f2da..f40b0d7e086 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -1513,18 +1513,17 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { routingPolicy: "board_escalation_no_takeover_v1", }); if (input.cause === "execution_review_participant_recovery") { - expect(action.nextAction).toContain("failed review participant path"); + expect(action.nextAction).toContain("The reviewer did not finish the review"); } else if (input.cause === "process_lost") { - expect(action.nextAction).toContain( - "explicitly retry the original owner", - ); + expect(action.nextAction).toContain("Press Retry to start it again from the saved work"); } else { expect(action.nextAction).toContain( - input.kind === "deliberate_wait_without_target" ? "repair, retry the original owner" : input.kind === "missing_disposition" - ? "valid issue disposition" - : "Board operator", + input.kind === "deliberate_wait_without_target" ? "without saying what happens next" : input.kind === "missing_disposition" + ? "without saying whether the task is done" + : "Press Retry to start it again from the saved work", ); } + expect(action.nextAction).not.toMatch(/Board operator|inspect/i); const recoveryIssues = await db .select() @@ -6589,7 +6588,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { status: "active", ownerType: "board", }); - expect(recoveryAction?.nextAction).toContain("sandbox provider plugin"); + expect(recoveryAction?.nextAction).toContain("plugin that starts this agent's sandbox is not running"); expect(recoveryAction?.nextAction).toContain("enable the plugin"); const notice = await waitForValue(async () => { @@ -7144,14 +7143,14 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { recoveryCause: "workspace_validation_failed", }); expect(recoveryAction?.nextAction).toContain( - "repair the source task workspace link", + "The task's workspace could not be used", ); const validationComment = await waitForValue(async () => { const rows = await selectRecoveryCommentsForIssue(issueId); return ( rows.find((comment) => - comment.body.includes("workspace failed validation"), + comment.body.includes("workspace could not be used"), ) ?? null ); }); @@ -7199,7 +7198,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(action).toMatchObject({ status: "active", kind: "workspace_validation", ownerType: "board" }); expect(action.nextAction).toContain("workspace"); const comments = await selectRecoveryCommentsForIssue(issueId); - expect(comments.some((comment) => comment.body.includes("workspace failed validation"))).toBe(true); + expect(comments.some((comment) => comment.body.includes("workspace could not be used"))).toBe(true); expect(mockCaptureRunFailure.mock.calls.filter(([event]) => event.runId === runId)).toEqual([]); } finally { await heartbeat.waitForRunExecutionDrain(runId); @@ -7293,7 +7292,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(action).toMatchObject({ status: "active", kind: "workspace_validation", ownerType: "board" }); expect(action.nextAction).toContain("workspace"); const comments = await selectRecoveryCommentsForIssue(issueId); - expect(comments.some((comment) => comment.body.includes("workspace failed validation"))).toBe(true); + expect(comments.some((comment) => comment.body.includes("workspace could not be used"))).toBe(true); expect(mockCaptureRunFailure.mock.calls.filter(([event]) => event.runId === runId)).toHaveLength(report ? 1 : 0); const calls = (await fs.readFile(callsPath, "utf8")).trim().split("\n").map((line) => JSON.parse(line) as string[]); expect(calls).toHaveLength(failures.length); @@ -7516,13 +7515,13 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { ownerAgentId: null, recoveryIssueId: null, }); - expect(recoveryAction?.nextAction).toContain("bind the missing secret"); + expect(recoveryAction?.nextAction).toContain("A secret this agent needs is not set"); const configurationComment = await waitForValue(async () => { const rows = await selectRecoveryCommentsForIssue(issueId); return ( rows.find((comment) => - comment.body.includes("secret/env bindings are missing"), + comment.body.includes("a secret it needs is not set"), ) ?? null ); }); @@ -7881,7 +7880,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect.objectContaining({ type: "key_value", label: "Recovery owner", - value: "Board decision required", + value: "Waiting for you", }), ]), }), @@ -11573,7 +11572,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { comments.find( (comment) => comment.body.includes( - "pending execution-review participant once", + "did not finish the review, even after Paperclip asked again", ) && noticeMetadataReferencesRecoveryAction( comment.metadata, @@ -12401,7 +12400,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { (row) => row.type === "key_value" && row.label === "Recovery owner" && - row.value === "Board decision required", + row.value === "Waiting for you", ), ).toBe(true); }); @@ -12877,7 +12876,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { (row) => row.type === "key_value" && row.label === "Recovery owner" && - row.value === "Board decision required", + row.value === "Waiting for you", ), ).toBe(true); }); @@ -14935,7 +14934,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { "stopped automatic stranded-work recovery", ); expect(comments[0]?.body).toContain( - "Latest retry failure details were withheld from the issue thread", + "The failure details are kept with the run", ); expect(comments[0]?.body).toContain( "recovery issues do not create nested `stranded_issue_recovery` issues", diff --git a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts index fc0b86f7125..a167634051b 100644 --- a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts +++ b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts @@ -712,7 +712,7 @@ async function expectContainedWorkspaceBranchFailure(input: { }), }), }), - nextAction: expect.stringContaining("choose a new execution workspace"), + nextAction: expect.stringContaining("workspace no longer matches its branch"), wakePolicy: expect.objectContaining({ type: "board_escalation", reason: "workspace_validation_failed", diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts index b6896998808..372cd77ba5e 100644 --- a/server/src/__tests__/issue-recovery-actions.test.ts +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -1856,7 +1856,7 @@ describeEmbeddedPostgres("issue recovery actions", () => { cleanliness: "dirty", }), }), - nextAction: expect.stringContaining("git worktree branch incoherence"), + nextAction: expect.stringContaining("workspace no longer matches its branch"), wakePolicy: expect.objectContaining({ type: "board_escalation", reason: "workspace_validation_failed", diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index ef2894a8cd0..91d1bbca4cb 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -39,6 +39,7 @@ import { firstTurnDeliveredCommentIds, historyReadDeliveredCommentIds } from "./ import { canResumePreservedResultFinalization } from "./preserved-result-finalization.js"; import { isServerReplacementInterruption } from "./server-replacement-interruption.js"; import { normalizeAgentNameKey } from "./agent-name-key.js"; +import { liftInterruptionBlock } from "./interruption-block-release.js"; type Run = Pick /** A reconciliation hold that blocks the task and that this decision settles. */ function isOwnedReconciliationHold(hold: typeof issueRecoveryActions.$inferSelect) { + if (hold.kind === "stranded_assigned_issue") return false; if (!(EXECUTION_RECONCILIATION_CAUSES as readonly string[]).includes(hold.cause) || !blocksExecution(hold)) return false; if (isHumanGate(hold)) return false; // A restore hold is owned only once its saved-work recovery completed. diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index e54dd59825f..65a6a15244f 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -512,7 +512,8 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { } it.each([ - ["issue_continuation_needed", "cancelled"], ["issue_recovery_action_restored", "cancelled"], ["issue_commented", "running"], + ["issue_continuation_needed", "cancelled"], ["issue_recovery_action_restored", "cancelled"], ["issue_children_completed", "cancelled"], + ["issue_commented", "running"], ] as const)("does not start a %s run whose task another run already finished (expected %s)", async (wakeReason, expected) => { // Release-swap wave t011: a delivered continuation queued while its task was open started after // another run marked the task done, and took an extra turn on a finished task. diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 1be0c728e84..ec362ee5f17 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -909,7 +909,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende /** System continuations that only make sense for an open task. */ const SYSTEM_CONTINUATION_WAKE_REASONS = new Set(["issue_recovery_action_restored", "issue_continuation_needed", - "issue_assignment_recovery", ISSUE_BLOCKER_STALLED_WAKE_REASON]); + "issue_assignment_recovery", ISSUE_BLOCKER_STALLED_WAKE_REASON, "issue_children_completed"]); /** * Cancels a system continuation whose task closed while it was queued, @@ -985,9 +985,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // A system continuation queued while its task was open has nothing left to // do once another run finished or cancelled the task before this one // started. A person's message on a done task is not one of these reasons. - const systemContinuationReason = readNonEmptyString(parseObject(run.contextSnapshot).wakeReason); - const continuedIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); - if (systemContinuationReason && SYSTEM_CONTINUATION_WAKE_REASONS.has(systemContinuationReason) && continuedIssueId) { + const runContext = parseObject(run.contextSnapshot); + const systemContinuationReason = readNonEmptyString(runContext.wakeReason); + const continuedIssueId = readNonEmptyString(runContext.issueId); + if (systemContinuationReason && SYSTEM_CONTINUATION_WAKE_REASONS.has(systemContinuationReason) && continuedIssueId + && runContext.onboardingCompletion !== true) { const [target] = await db.select({ status: issues.status }).from(issues) .where(and(eq(issues.companyId, run.companyId), eq(issues.id, continuedIssueId))); if (target && ["done", "cancelled"].includes(target.status)) { diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index 8e562cdcd3e..3ae6522b205 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -1792,7 +1792,7 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( expect(actions[0]).toMatchObject({ status: "active", ownerType: "board", cause: "remote_workspace_restore_failed", kind: "stranded_assigned_issue", returnOwnerAgentId: f.agent.id }); expect(actions[0]!.evidence).toMatchObject({ runId: f.run.id, restoreFailure: failed.error }); - expect(actions[0]!.nextAction).toContain("saved workspace version was kept"); + expect(actions[0]!.nextAction).toContain("The saved version was kept"); }); it("keeps an active recovery action that another writer created for the task", async () => { diff --git a/server/src/services/interruption-block-release.ts b/server/src/services/interruption-block-release.ts new file mode 100644 index 00000000000..f8ffdd77204 --- /dev/null +++ b/server/src/services/interruption-block-release.ts @@ -0,0 +1,64 @@ +import { and, desc, eq, sql, type SQLWrapper } from "drizzle-orm"; +import { activityLog, heartbeatRuns, issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db"; +import { holdOwnsBlockedStatus } from "./recovery-hold-blocked-status.js"; +import { persistActivity } from "./activity-log.js"; + +export const BLOCK_ATTRIBUTION_WINDOW_MS = 10_000; + +type BlockEntry = { actorType: string; actorId: string; agentId: string | null; details: unknown }; + +const object = (value: unknown): Record => + value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + +export function blockedStatusDetailsSql(details: SQLWrapper) { + return sql`(${details}->>'status' = 'blocked' or ${details}->>'toStatus' = 'blocked' + or ${details}->'changes'->'status'->>'to' = 'blocked')`; +} + +export function attributeBlock(entry: BlockEntry | undefined, + assigneeAgentId: string | null): "user" | "assignee" | "system" | "unknown" { + if (!entry) return "unknown"; + const details = object(entry.details); + if (entry.actorType === "user") return "user"; + if (entry.actorType === "agent") return entry.agentId && entry.agentId === assigneeAgentId ? "assignee" : "system"; + if (entry.actorId === "native-status-committer" && details.source === "native_status_decision" + && details.reasonCode !== "durable_dependency_blocker_bound") return "assignee"; + return "system"; +} + +type SourceRun = Pick; + +export async function liftInterruptionBlock(db: Db, task: typeof issues.$inferSelect, + actions: Array, sources: SourceRun[], successorRunId: string) { + if (task.status !== "blocked" || task.unblockDescriptor) return; + const sourceFor = (action: typeof issueRecoveryActions.$inferSelect) => + sources.find(run => run.id === String(action.evidence.runId ?? action.evidence.sourceRunId)); + const blockedAt = task.blockedTransitionAt?.getTime(); + const receiptless = actions.every(action => + action.evidence.workspaceFinalizationBlock === undefined && action.evidence.nativeFailureBlock === undefined); + const owned = actions.some(action => { const run = sourceFor(action); return run && holdOwnsBlockedStatus(action, task, run); }) + || (receiptless && task.lastStatusDecisionId === null && blockedAt !== undefined + && sources.some(run => run.finishedAt !== null && blockedAt >= run.finishedAt.getTime() - 1_000)); + if (!owned) return; + const [blockEntry] = blockedAt === undefined ? [] : await db.select({ actorType: activityLog.actorType, + actorId: activityLog.actorId, agentId: activityLog.agentId, details: activityLog.details }).from(activityLog).where(and( + eq(activityLog.companyId, task.companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, task.id), + sql`${activityLog.createdAt} between ${new Date(blockedAt - BLOCK_ATTRIBUTION_WINDOW_MS).toISOString()}::timestamptz + and ${new Date(blockedAt + BLOCK_ATTRIBUTION_WINDOW_MS).toISOString()}::timestamptz`, + blockedStatusDetailsSql(activityLog.details), + )).orderBy(desc(activityLog.createdAt)).limit(1); + if (attributeBlock(blockEntry, task.assigneeAgentId) === "user") return; + const [openBlocker] = await db.select({ id: issueRelations.id }).from(issueRelations) + .innerJoin(issues, and(eq(issues.id, issueRelations.issueId), eq(issues.companyId, issueRelations.companyId))) + .where(and(eq(issueRelations.companyId, task.companyId), eq(issueRelations.relatedIssueId, task.id), + eq(issueRelations.type, "blocks"), sql`${issues.status} not in ('done', 'cancelled')`)).limit(1); + if (openBlocker) return; + const lifted = await db.update(issues).set({ status: "todo", blockedTransitionAt: null, blockedOwnerNotifiedAt: null, + updatedAt: new Date() }).where(and(eq(issues.companyId, task.companyId), eq(issues.id, task.id), + eq(issues.status, "blocked"), eq(issues.statusVersion, task.statusVersion))).returning({ id: issues.id }); + if (!lifted.length) return; + await persistActivity(db, { companyId: task.companyId, actorType: "system", actorId: "task-recovery", + action: "issue.updated", entityType: "issue", entityId: task.id, + details: { status: "todo", previousStatus: "blocked", reason: "continuation_queued", successorRunId, + _previous: { status: "blocked" } } }); +} diff --git a/server/src/services/recovery/assigned-work-invariant.test.ts b/server/src/services/recovery/assigned-work-invariant.test.ts index 00b9aaff2a0..6f56b1e550b 100644 --- a/server/src/services/recovery/assigned-work-invariant.test.ts +++ b/server/src/services/recovery/assigned-work-invariant.test.ts @@ -61,6 +61,19 @@ describe("assigned-work invariant", () => { .toEqual({ kind: "violation", reason: "native_run_stalled", runId: "n", stale: false }); }); + it("calls a recovery block with a live run for the same task inconsistent, and keeps someone else's block", () => { + const queued = [{ id: "q", status: "queued", scheduledRetryAt: null, owner: "alive" as const }]; + expect(evaluateAssignedWork(facts({ issue: { status: "blocked", blockedTransitionAt: ago(60_000) }, blockedBy: "system", activeRuns: queued }), now)) + .toEqual({ kind: "violation", reason: "blocked_with_live_run", runId: "q", stale: false }); + expect(evaluateAssignedWork(facts({ issue: { status: "blocked" }, blockedBy: "unknown", activeRuns: queued }), now)) + .toMatchObject({ kind: "violation", reason: "blocked_with_live_run" }); + for (const other of [{ blockedBy: "user" as const }, { openBlockers: 1 }, { pendingApprovals: 1 }, + { blockedBy: "system" as const, issue: { status: "blocked", unblockOwner: { userId: "u" } } }]) { + expect(evaluateAssignedWork(facts({ issue: { status: "blocked" }, activeRuns: queued, ...other }), now)) + .toEqual({ kind: "ok", clause: "live_run", runId: "q" }); + } + }); + it("does not count a running run whose owner is proven gone", () => { expect(evaluateAssignedWork(facts({ activeRuns: [{ id: "r", status: "running", scheduledRetryAt: null, owner: "gone" }] }), now)) .toEqual({ kind: "violation", reason: "run_owner_gone", runId: "r", stale: false }); diff --git a/server/src/services/recovery/assigned-work-invariant.ts b/server/src/services/recovery/assigned-work-invariant.ts index 9bc465ea4e4..4c459622433 100644 --- a/server/src/services/recovery/assigned-work-invariant.ts +++ b/server/src/services/recovery/assigned-work-invariant.ts @@ -63,6 +63,7 @@ export const VIOLATION_REASONS = [ "recovery_block", /** Blocked, and nothing recorded says why. */ "unknown_block", + "blocked_with_live_run", ] as const; export type ViolationReason = (typeof VIOLATION_REASONS)[number]; @@ -158,7 +159,8 @@ export type AssignedWorkFacts = { lastBlockerResolvedAt?: Date | null; openChildren: number; reviewPending: boolean; - activeRecoveryAction: { cause: string; ownerType: string; resumeAt: Date | null } | null; + activeRecoveryAction: { cause: string; ownerType: string; resumeAt: Date | null; + continuedBySweep?: boolean; id?: string } | null; agent: { status: string; invokable: boolean } | null; companyPaused: boolean; pauseHold: boolean; @@ -187,7 +189,12 @@ export function evaluateAssignedWork(facts: AssignedWorkFacts, now: Date = new D // fails closed: during a deploy overlap another server may still own it. const live = facts.activeRuns.find(run => (run.status === "queued" || run.status === "running") && run.owner !== "gone" && run.owner !== "stalled"); - if (live) return { kind: "ok", clause: "live_run", runId: live.id }; + if (live) { + if (issue.status === "blocked" && !waitingReason(facts, now)) { + return { kind: "violation", reason: "blocked_with_live_run", runId: live.id, stale: false }; + } + return { kind: "ok", clause: "live_run", runId: live.id }; + } // A run row still says running but its server is proven gone. It holds the // task's execution lock, so nothing pending can start until it is settled. @@ -336,6 +343,8 @@ export function continuationActivityMessage(reason: ViolationReason): string { return "Continued automatically after an interruption. Nothing needed you, so Paperclip queued the task again."; case "retry_offered": return "Continued automatically after an interruption. Paperclip queued the task again, so you don't need to press Retry."; + case "blocked_with_live_run": + return "No longer blocked: this task is already queued to continue, so nothing needs you."; default: return "Continued automatically after an interruption. Nothing was running or scheduled, so Paperclip queued the task again."; } diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 2c5452b71c1..703348cb5f7 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -30,6 +30,8 @@ import { taskInterruptionPresentations } from "../task-interruption-presentation import { CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js"; import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js"; import { PENDING_CONTINUATION_BOUND_MS } from "../recovery-continuation-pacing.js"; +import { attributeBlock, BLOCK_ATTRIBUTION_WINDOW_MS, blockedStatusDetailsSql } from "../interruption-block-release.js"; +import { STRANDED_CONTINUATION_MAX_AGE_MS, STRANDED_KIND, strandedHoldContinuesAutomatically, type StrandedHoldRun } from "./stranded-hold-continuation.js"; import { continuationActivityMessage, continuationAllowance, @@ -50,6 +52,7 @@ import { * durable claim two servers serialize on, and the per-task rate-limit ledger. */ export const ASSIGNED_WORK_CONTINUED_ACTION = "issue.continued_automatically"; export const ASSIGNED_WORK_CONTINUATION_SOURCE = "issue.assigned_work_invariant"; +export const STRANDED_TRANSIENT_CONTINUED_OUTCOME = "stranded_transient_continued"; /** Repairs per process per minute, so a deploy with many violations does not * start hundreds of runs at once. The rest are deferred, newest first. */ export const ASSIGNED_WORK_REPAIRS_PER_MINUTE = 10; @@ -63,7 +66,6 @@ export { PENDING_CONTINUATION_BOUND_MS }; const NON_TERMINAL_RUN_STATUSES = ["queued", "running", "scheduled_retry"]; const NATIVE_PENDING_RECOVERY_STATES = ["awaiting_evidence", "awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"]; const PROVIDER_LIMIT_CAUSES = ["provider_quota", "native_provider_usage_limit"]; -const BLOCK_ATTRIBUTION_WINDOW_MS = 10_000; type IssueRow = Pick ${BLOCK_ATTRIBUTION_WINDOW_MS / 1000}) and requested.blocked_at + make_interval(secs => ${BLOCK_ATTRIBUTION_WINDOW_MS / 1000}) - and (a.details->>'status' = 'blocked' or a.details->>'toStatus' = 'blocked' - or a.details->'changes'->'status'->>'to' = 'blocked') + and ${blockedStatusDetailsSql(sql`a.details`)} order by a.created_at desc limit 1 ) latest`) : [], taskInterruptionPresentations(database, companyId, issueRows.map(issue => ({ id: issue.id, companyId: issue.companyId, @@ -276,6 +277,14 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { ]); const rowsOf = (value: unknown): T[] => Array.isArray(value) ? value as T[] : ((value as { rows?: T[] })?.rows ?? []); + const strandedRunIds = [...actionMap.values()].filter(action => action.kind === STRANDED_KIND && action.ownerType === "board") + .map(action => action.evidence?.latestRunId).filter((id): id is string => typeof id === "string" && /^[0-9a-f-]{36}$/i.test(id)); + const strandedRuns = new Map((strandedRunIds.length ? await database.select({ id: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode, error: heartbeatRuns.error, + finishedAt: heartbeatRuns.finishedAt, + executionRecovery: sql`${heartbeatRuns.resultJson}->'executionRecovery'` }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, strandedRunIds))) : []) + .map(({ id, ...run }) => [id, run])); const agentsById = new Map(); for (const agent of agentRows as Array) { // An agent still being set up, or held by a lifecycle hold, takes no work @@ -372,8 +381,10 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { lastBlockerResolvedAt: resolvedBlockerAt.get(issue.id) ?? null, openChildren: childCounts.get(issue.id) ?? 0, reviewPending: executionState.status === "pending" || Boolean(executionState.reviewRequest), - activeRecoveryAction: action ? { cause: action.cause, ownerType: action.ownerType, - resumeAt: PROVIDER_LIMIT_CAUSES.includes(action.cause) && interruption?.nextRetryAt ? new Date(interruption.nextRetryAt) : null } : null, + activeRecoveryAction: action ? { id: action.id, cause: action.cause, ownerType: action.ownerType, + resumeAt: PROVIDER_LIMIT_CAUSES.includes(action.cause) && interruption?.nextRetryAt ? new Date(interruption.nextRetryAt) : null, + continuedBySweep: strandedHoldContinuesAutomatically(action, + strandedRuns.get(String(action.evidence?.latestRunId ?? "")) ?? null, issue.assigneeAgentId, at) } : null, agent: agent ? { status: agent.status, invokable: agent.invokable } : null, companyPaused: company?.status === "paused", pauseHold, @@ -415,6 +426,11 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { */ async function listSuspects(scope: { companyId?: string; issueId?: string; createdAfter?: Date | null }, cursor: string | null, limit: number) { const at = now(); + const liveRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} + and (r.context_snapshot->>'issueId') = ${issues.id}::text + and r.status in ('queued', 'running', 'scheduled_retry') + and not (r.status = 'running' and r.runtime_mode = 'legacy' and r.controller_lease_expires_at is not null + and r.controller_lease_expires_at < ${at.toISOString()}::timestamptz))`; return db.select({ id: issues.id, companyId: issues.companyId, updatedAt: issues.updatedAt }).from(issues).where(and( scope.companyId ? eq(issues.companyId, scope.companyId) : undefined, scope.issueId ? eq(issues.id, scope.issueId) : undefined, @@ -430,17 +446,13 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { sql`(${issues.monitorNextCheckAt} is null or ${issues.monitorNextCheckAt} <= ${at.toISOString()}::timestamptz)`, // A legacy run whose controller lease lapsed may have lost its owner; // the predicate judges it with the owner proof. Other active runs are live. - sql`not exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} - and (r.context_snapshot->>'issueId') = ${issues.id}::text - and r.status in ('queued', 'running', 'scheduled_retry') - and not (r.status = 'running' and r.runtime_mode = 'legacy' and r.controller_lease_expires_at is not null - and r.controller_lease_expires_at < ${at.toISOString()}::timestamptz))`, - sql`not exists (select 1 from agent_wakeup_requests w where w.company_id = ${issues.companyId} + sql`(${issues.status} = 'blocked' or not ${liveRun})`, + sql`((${issues.status} = 'blocked' and ${liveRun}) or not exists (select 1 from agent_wakeup_requests w where w.company_id = ${issues.companyId} and (w.payload->>'issueId') = ${issues.id}::text and (w.status = 'queued' or (w.status = 'deferred_issue_execution' and w.created_at > ${new Date(at.getTime() - DEFERRED_WAKE_PENDING_BOUND_MS).toISOString()}::timestamptz) or (w.status = 'claimed' and (w.run_id is not null - or coalesce(w.claimed_at, w.created_at) > ${new Date(at.getTime() - DEFERRED_WAKE_PENDING_BOUND_MS).toISOString()}::timestamptz))))`, + or coalesce(w.claimed_at, w.created_at) > ${new Date(at.getTime() - DEFERRED_WAKE_PENDING_BOUND_MS).toISOString()}::timestamptz)))))`, sql`not exists (select 1 from issue_thread_interactions t where t.company_id = ${issues.companyId} and t.issue_id = ${issues.id} and t.status = 'pending' and ${interactionNotSupersededSql(sql`t.company_id`, sql`t.issue_id::text`, sql`t.created_at`)})`, @@ -456,7 +468,11 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { // hold on a blocked task is re-continued by the hold's owner. sql`not exists (select 1 from issue_recovery_actions ra where ra.company_id = ${issues.companyId} and ra.source_issue_id = ${issues.id} and ra.status in ('active', 'escalated') - and (ra.cause in ('configuration_incomplete', 'ownership_owner_unprovable') or ${issues.status} = 'blocked'))`, + and (ra.cause in ('configuration_incomplete', 'ownership_owner_unprovable') or ${issues.status} = 'blocked') + -- except a recent board escalation, which the predicate judges by its failed run's cause + and not (ra.kind = ${STRANDED_KIND} and ra.owner_type = 'board' and exists (select 1 from heartbeat_runs sr + where sr.company_id = ra.company_id and sr.id::text = ra.evidence->>'latestRunId' + and sr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz)))`, // An assignee that cannot take work (paused, being set up, held) is // its own wait; the predicate names it when anything else asks. sql`exists (select 1 from agents a where a.id = ${issues.assigneeAgentId} and a.company_id = ${issues.companyId} @@ -485,11 +501,15 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { if (verdict.kind === "not_applicable" || verdict.kind === "ok") return { verdict, outcome: "ok", retryAt: null }; if (verdict.kind === "settling") return { verdict, outcome: "settling", retryAt: verdict.recheckAt }; if (verdict.stale) return { verdict, outcome: "stale_reported", retryAt: null }; + if (verdict.reason === "blocked_with_live_run") { + return { verdict, outcome: await liftBlockBehindLiveRun(issue, at) ? "repaired" : "changed", retryAt: null }; + } // A task blocked under an active recovery hold is re-continued by the // hold's owner (the fresh-continuation fallback, same per-task cap). The // sweep reports it and repairs only blocks that have no hold at all, so - // the two never continue one task twice. - if (issue.status === "blocked" && facts.activeRecoveryAction) return { verdict, outcome: "left_to_hold_owner", retryAt: null }; + if (issue.status === "blocked" && facts.activeRecoveryAction && !facts.activeRecoveryAction.continuedBySweep) { + return { verdict, outcome: "left_to_hold_owner", retryAt: null }; + } // A stalled native run is ended by native recovery, not here: the sweep // builds no second ender. It stays a counted violation in the report. if (verdict.reason === "native_run_stalled") return { verdict, outcome: "left_to_native_recovery", retryAt: null }; @@ -561,7 +581,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { const fresh = (await loadFacts(issue.companyId, [current], { database: tx as unknown as Db, proveOwners: false })).get(current.id)!; const verdict = evaluateAssignedWork(fresh, at); if (verdict.kind !== "violation" || verdict.stale || verdict.reason === "run_owner_gone" || verdict.reason === "native_run_stalled" - || (current.status === "blocked" && fresh.activeRecoveryAction)) return { kind: "changed" as const }; + || (current.status === "blocked" && fresh.activeRecoveryAction && !fresh.activeRecoveryAction.continuedBySweep)) return { kind: "changed" as const }; const recent = await tx.select({ createdAt: activityLog.createdAt }).from(activityLog).where(and( eq(activityLog.companyId, issue.companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issue.id), eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), @@ -578,6 +598,14 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { sql`(${activityLog.details}->>'status' = 'blocked' or ${activityLog.details}->>'toStatus' = 'blocked')`, )).orderBy(desc(activityLog.createdAt)).limit(1); restoredTo = recordedPreviousStatus(blocking?.details); + const hold = fresh.activeRecoveryAction; + if (hold?.continuedBySweep && hold.id) { + await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "restored", resolvedAt: at, updatedAt: at, + ownerType: "system", ownerAgentId: null, ownerUserId: null, returnOwnerAgentId: issue.assigneeAgentId, + resolutionNote: STRANDED_TRANSIENT_CONTINUED_OUTCOME, nextAction: CONTINUATION_PENDING_MESSAGE, wakePolicy: null, monitorPolicy: null }) + .where(and(eq(issueRecoveryActions.companyId, issue.companyId), eq(issueRecoveryActions.id, hold.id), + inArray(issueRecoveryActions.status, ["active", "escalated"]))); + } await issueService(tx as unknown as Db).update(issue.id, { status: restoredTo }, tx); const [restored] = await tx.select({ statusVersion: issues.statusVersion }).from(issues).where(eq(issues.id, issue.id)); restoredStatusVersion = restored?.statusVersion ?? null; @@ -597,6 +625,30 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { }); } + async function liftBlockBehindLiveRun(issue: IssueRow, at: Date): Promise { + const publications: ActivityPublication[] = []; + const lifted = await db.transaction(async tx => { + const [current] = await tx.select(issueColumns).from(issues) + .where(and(eq(issues.companyId, issue.companyId), eq(issues.id, issue.id))).for("update"); + if (!current || current.status !== "blocked" || current.statusVersion !== issue.statusVersion) return false; + const fresh = (await loadFacts(issue.companyId, [current], { database: tx as unknown as Db, proveOwners: false })).get(current.id)!; + const verdict = evaluateAssignedWork(fresh, at); + if (verdict.kind !== "violation" || verdict.reason !== "blocked_with_live_run") return false; + const running = fresh.activeRuns.some(run => run.id === verdict.runId && run.status === "running"); + const status = running ? "in_progress" : "todo"; + await issueService(tx as unknown as Db).update(issue.id, { status }, tx); + await logActivity(tx as unknown as Db, { + companyId: issue.companyId, actorType: "system", actorId: "system", agentId: issue.assigneeAgentId, runId: null, + action: "issue.updated", entityType: "issue", entityId: issue.id, + details: { identifier: issue.identifier, status, previousStatus: "blocked", source: ASSIGNED_WORK_CONTINUATION_SOURCE, + violation: verdict.reason, message: continuationActivityMessage(verdict.reason) }, + }, publications); + return true; + }); + if (lifted) for (const publication of publications) publishActivity(publication); + return lifted; + } + /** Put back a block lifted for a continuation that admission refused, when * nothing changed the task since: same status version, nothing pending. */ async function revertRestoredBlock(issue: IssueRow, claimed: Claimed) { @@ -665,7 +717,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { else if (verdict.kind === "violation") { if (verdict.stale) { summary.staleBlocked += 1; continue; } summary.violationCount += 1; - if (issue.status === "blocked" && facts.get(issue.id)!.activeRecoveryAction) summary.heldByRecoveryOwner += 1; + const hold = facts.get(issue.id)!.activeRecoveryAction; + if (issue.status === "blocked" && hold && !hold.continuedBySweep) summary.heldByRecoveryOwner += 1; summary.violations[verdict.reason] = (summary.violations[verdict.reason] ?? 0) + 1; if (summary.sample.length < 20) summary.sample.push({ identifier: issue.identifier, status: issue.status, reason: verdict.reason }); } @@ -701,20 +754,6 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { return { checkIssue, listSuspects, report, loadFacts, flushPassLog: passLog.flush }; } -/** Who moved a task to `blocked`, from the activity entry written with it. */ -export function attributeBlock(entry: { actorType: string; actorId: string; agentId: string | null; details: unknown } | undefined, - assigneeAgentId: string | null): "user" | "assignee" | "system" | "unknown" { - if (!entry) return "unknown"; - const details = json(entry.details); - if (entry.actorType === "user") return "user"; - if (entry.actorType === "agent") return entry.agentId && entry.agentId === assigneeAgentId ? "assignee" : "system"; - // The agent's own turn decided to block (committed by the native status - // decision), except when the decision only bound a dependency. - if (entry.actorId === "native-status-committer" && details.source === "native_status_decision" - && details.reasonCode !== "durable_dependency_blocker_bound") return "assignee"; - return "system"; -} - /** A pending interaction no explicit user continuation (Retry or message) * started after it. Uses the (company, issueId, created) run index. */ function interactionNotSupersededSql(companyId: SQLWrapper, issueIdText: SQLWrapper, createdAt: SQLWrapper) { diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 12834ba7746..7e786b071f6 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -101,6 +101,7 @@ import { workspaceOperations, } from "@paperclipai/db"; import { parseObject, asBoolean, asNumber } from "../../adapters/utils.js"; +import { STRANDED_KIND, strandedRunCauseIsTemporary, type StrandedHoldRun } from "./stranded-hold-continuation.js"; import { runningProcesses } from "../../adapters/index.js"; import { isNativeRunnerOwnershipHeld, @@ -174,6 +175,7 @@ import { buildExecutionReviewParticipantRecoveryNoticeSeed, buildExecutionReviewParticipantUnavailableNoticeSeed, buildStrandedRecoveryEscalationNotice, + RETRY_FROM_SAVED_WORK_NEXT_ACTION, type StrandedRecoveryNoticeSeed, } from "./stranded-notice.js"; import { @@ -530,7 +532,7 @@ function summarizeRunFailureForIssueComment(run: LatestIssueRun) { if (!run) return null; if (readNonEmptyString(run.error) || readNonEmptyString(run.errorCode)) { - return " Latest retry failure details were withheld from the issue thread; inspect the linked run for evidence."; + return " The failure details are kept with the run."; } return null; } @@ -2515,7 +2517,7 @@ export function recoveryService( "", "- Nested recovery: suppressed because this issue is already a `stranded_issue_recovery` issue.", sourceLine, - "- Next action: the assigned recovery owner or board operator should fix the runtime/adapter problem, resolve or reassign the original source issue, then mark this recovery issue done or cancelled.", + "- Next action: press Retry on the original task to start it again from the saved work, then close this task.", ].join("\n"); } @@ -2679,43 +2681,43 @@ export function recoveryService( : { routingPolicy: STRANDED_BOARD_ESCALATION_POLICY }, nextAction: recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON - ? "Board operator: inspect the run evidence, then explicitly choose a valid issue disposition, retry the original owner, reassign, or intentionally resolve the task." + ? "The agent finished without saying whether the task is done. Mark the task done if it is, or press Retry to let the agent continue from the saved work." : recoveryCause === "process_lost" - ? "Board operator: inspect the retry history, then explicitly retry the original owner, reassign, or intentionally resolve the task." + ? RETRY_FROM_SAVED_WORK_NEXT_ACTION : recoveryCause === "provider_quota" - ? "Wait for provider quota recovery, then retry the original assignee; do not wake a takeover owner." + ? "The AI provider's usage limit was reached. The same agent continues the task once the limit resets." : isEnvironmentProviderWait - ? "No agent work started. Retry the task once the environment provider is available." + ? "The sandbox service was unavailable, so no agent work started. Press Retry once it is available again." : recoveryCause === "codex_output_inactivity_monitor" - ? "Board operator: inspect the inactivity evidence, then explicitly retry the original owner, reassign, or intentionally resolve the task." + ? "The agent stopped responding, so this attempt was ended. Press Retry to start it again from the saved work." : recoveryCause === "workspace_validation_failed" ? readWorkspaceValidationPayload(input.latestRun)?.reason === "git_worktree_branch_incoherence" - ? "Board operator: repair the source task git worktree branch incoherence or choose a new execution workspace, then explicitly retry or reassign." + ? "The task's workspace no longer matches its branch, so the agent did not start. Press Retry to start it again." : readWorkspaceValidationPayload(input.latestRun) ?.reason === "git_worktree_base_materialization_failed" - ? "Board operator: repair the project workspace repository URL or clone access, or configure a local checkout cwd, then explicitly retry or reassign." - : "Board operator: repair the source task workspace link, project workspace cwd, or git checkout, then explicitly retry or reassign." + ? "Paperclip could not copy the project's repository into the task's workspace. Restore access to the repository in the project settings, then press Retry." + : "The task's workspace could not be used, so the agent did not start. Press Retry to start it again." : recoveryCause === "configuration_incomplete" ? readConfigurationIncompletePayload(input.latestRun)?.reason === "workspace_base_ref_unresolved" - ? "Check the starting branch and repository access, then repair the task’s branch and retry the original assignee." + ? "The starting branch for this task's workspace was not found in the repository. Set a starting branch that exists, then press Retry." : readConfigurationIncompletePayload(input.latestRun)?.reason === "ai_connection_unavailable" ? "Reconnect the selected AI account or choose an available connection, then continue the task." : readConfigurationIncompletePayload(input.latestRun)?.reason === RESTRICTED_WORKSPACE_SOURCE_REASON ? "Retry the task." : readConfigurationIncompletePayload(input.latestRun) ?.reason === SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON - ? `Board operator: the sandbox provider plugin named in the run failure is not ready; ${sandboxProviderPluginRemedy( + ? `The plugin that starts this agent's sandbox is not running. To fix it, ${sandboxProviderPluginRemedy( readNonEmptyString( readConfigurationIncompletePayload(input.latestRun) ?.pluginStatus, ) ?? "error", - )}, then explicitly retry the original owner or reassign.` - : "Board operator: bind the missing secret(s) named in the run failure, then explicitly retry the original owner or reassign." + )}, then press Retry.` + : "A secret this agent needs is not set. Add it in the agent's settings, then press Retry." : recoveryCause === "execution_review_participant_recovery" - ? "Board operator: repair the failed review participant path, restore a live reviewer, explicitly reassign, or record an intentional resolution." - : "Board operator: inspect the evidence, repair the runtime if appropriate, then explicitly retry the original owner, reassign, or intentionally resolve the task.", + ? "The reviewer did not finish the review. Choose a reviewer, or press Retry to ask the same reviewer again." + : RETRY_FROM_SAVED_WORK_NEXT_ACTION, wakePolicy: isProviderQuotaWait ? { type: "monitor_only", @@ -2891,7 +2893,7 @@ export function recoveryService( : "- Failure: none recorded", "- Guard: recovery issues do not create nested `stranded_issue_recovery` issues.", "", - "Next action: the current recovery owner should inspect the failed run evidence, restore a live execution path or record the manual resolution, then move this recovery issue out of `blocked`.", + "Next action: press Retry to start this task again from the saved work.", ].join("\n"); } @@ -3790,7 +3792,7 @@ export function recoveryService( routingPolicy: STRANDED_BOARD_ESCALATION_POLICY, }, nextAction: - "Inspect the evidence and choose whether to repair, retry the original owner, explicitly reassign, or resolve the source issue.", + "The agent stopped without saying what happens next with this task. Mark the task done if it is, or press Retry to let the agent continue from the saved work.", wakePolicy: { type: "board_escalation", reason: input.terminalReason, @@ -3825,7 +3827,7 @@ export function recoveryService( "- Recovery owner: board", "- Source ownership: unchanged; reassignment requires an explicit decision or a policy-defined serious failure.", "", - "Next action: repair the liveness disposition or request an explicit source-owner decision.", + "Next action: mark the task done if it is, or press Retry to let the agent continue from the saved work.", ].join("\n"), {}, { @@ -4292,6 +4294,14 @@ export function recoveryService( fallbackBody: input.comment, recoveryCause, recoveryActionId: recoveryAction.id, + continuesAutomatically: Boolean(input.latestRun) && recoveryAction.kind === STRANDED_KIND + && recoveryAction.ownerType === "board" && input.latestRun?.agentId === input.issue.assigneeAgentId + && strandedRunCauseIsTemporary({ + status: input.latestRun?.status ?? "", + errorCode: input.latestRun?.errorCode ?? null, + error: input.latestRun?.error ?? null, + executionRecovery: (parseObject(input.latestRun?.resultJson).executionRecovery ?? null) as StrandedHoldRun["executionRecovery"], + }), recoveryOwner: recoveryAction.ownerAgentId && recoveryOwner ? { id: recoveryOwner.id, name: recoveryOwner.name } diff --git a/server/src/services/recovery/stranded-hold-continuation.ts b/server/src/services/recovery/stranded-hold-continuation.ts new file mode 100644 index 00000000000..8951a820fff --- /dev/null +++ b/server/src/services/recovery/stranded-hold-continuation.ts @@ -0,0 +1,52 @@ +import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "../reusable-sandbox-resume-error.js"; + +export const STRANDED_CONTINUATION_MAX_AGE_MS = 24 * 60 * 60_000; +export const STRANDED_KIND = "stranded_assigned_issue"; + +const RESTART_CLASS_CODES = new Set([ + "process_lost", "server_shutdown_interrupted", "controller_lease_lost", "native_session_interrupted", + "native_runner_process_exited", "execution_finalization_deadline_exceeded", "local_runtime_api_listener_unavailable", +]); + +const TRANSIENT_SETUP_CODES = new Set([ + REUSABLE_SANDBOX_RESUME_FAILED_CODE, "remote_workspace_prepare_timeout", "environment_provider_unavailable", + "workspace_git_scan_timeout", "workspace_git_scan_saturated", "repository_catalog_unavailable", +]); + +const TRANSIENT_SANDBOX_CAUSES = new Set([ + "transport_timeout", "transport_closed", "sandbox_state_transition", "provider_rate_limited", "provider_unavailable", +]); + +export type StrandedHoldRun = { + agentId: string; + status: string; + errorCode: string | null; + error: string | null; + finishedAt: Date | null; + executionRecovery: { kind?: unknown; providerWorkStarted?: unknown } | null; +}; + +export function strandedHoldContinuesAutomatically( + hold: { kind: string; ownerType: string }, + run: StrandedHoldRun | null, + assigneeAgentId: string | null, + now: Date, +): boolean { + if (hold.kind !== STRANDED_KIND || hold.ownerType !== "board") return false; + if (!run || !assigneeAgentId || run.agentId !== assigneeAgentId || !run.finishedAt) return false; + if (now.getTime() - run.finishedAt.getTime() > STRANDED_CONTINUATION_MAX_AGE_MS) return false; + return strandedRunCauseIsTemporary(run); +} + +export function strandedRunCauseIsTemporary( + run: Pick, +): boolean { + if (!["failed", "timed_out", "interrupted", "cancelled"].includes(run.status)) return false; + const code = run.errorCode ?? ""; + if (RESTART_CLASS_CODES.has(code)) return true; + if (!TRANSIENT_SETUP_CODES.has(code)) return false; + if (run.executionRecovery?.kind !== "bootstrap" || run.executionRecovery.providerWorkStarted !== false) return false; + if (code !== REUSABLE_SANDBOX_RESUME_FAILED_CODE) return true; + const cause = /\(cause=([a-z_]+)/.exec(run.error ?? "")?.[1]; + return cause !== undefined && TRANSIENT_SANDBOX_CAUSES.has(cause); +} diff --git a/server/src/services/recovery/stranded-notice.test.ts b/server/src/services/recovery/stranded-notice.test.ts index 551187539f4..30b540b37e5 100644 --- a/server/src/services/recovery/stranded-notice.test.ts +++ b/server/src/services/recovery/stranded-notice.test.ts @@ -74,7 +74,7 @@ describe("stranded recovery notice seeds", () => { expect(seed.title).toBe("Shared workspace still held"); expect(seed.body).toContain("no output for more than an hour"); expect(seed.body).not.toContain("secret/env bindings"); - expect(seed.nextAction).toContain("stop it if it is stuck"); + expect(seed.nextAction).toContain("Stop the other run on this task"); }); it("offers retry without a workspace repair checklist", () => { @@ -86,9 +86,9 @@ describe("stranded recovery notice seeds", () => { it("keeps the secret-binding copy for other configuration gaps", () => { expect(buildConfigurationIncompleteRecoveryNoticeSeed({ reason: "secret_binding_missing" }).body).toContain( - "secret/env bindings", + "a secret it needs is not set", ); - expect(buildConfigurationIncompleteRecoveryNoticeSeed(null).body).toContain("secret/env bindings"); + expect(buildConfigurationIncompleteRecoveryNoticeSeed(null).body).toContain("a secret it needs is not set"); }); it("explains an unresolved workspace base ref and carries its remedy into the recovery card", () => { @@ -106,13 +106,13 @@ describe("stranded recovery notice seeds", () => { sourceRun: { id: "failed-run", status: "failed", errorCode: "configuration_incomplete" }, }); expect(notice.presentation.title).toBe("Workspace base ref unavailable"); - expect(notice.body).toContain("before the agent started"); - expect(notice.body).toContain("base ref"); - expect(notice.body).not.toContain("secret/env bindings"); + expect(notice.body).toContain("did not start"); + expect(notice.body).toContain("starting branch"); + expect(notice.body).not.toContain("secret"); expect(allRows(notice.metadata)).toContainEqual({ type: "key_value", label: "Next action", - value: "Check that the configured base ref exists and the repository is accessible. Correct the task or project workspace settings, then retry the task.", + value: "Set a starting branch that exists in the repository, then press Retry.", }); }); @@ -135,7 +135,25 @@ describe("buildStrandedRecoveryEscalationNotice", () => { expect(notice.presentation.title).toBe("Workspace scan timed out"); expect(notice.body).toContain("before the agent started"); expect(notice.body).not.toContain("retried continuation"); - expect(JSON.stringify(notice.metadata)).toContain("repository access and server load"); + expect(JSON.stringify(notice.metadata)).toContain("took too long, so no agent work started"); + }); + it("promises an automatic retry for a timed-out sandbox setup only when the sweep will continue it", () => { + for (const errorCode of ["reusable_sandbox_resume_failed", "remote_workspace_prepare_timeout"]) { + for (const continuesAutomatically of [true, false]) { + const notice = buildStrandedRecoveryEscalationNotice({ + seed: buildImmediateExecutionPathRecoveryNoticeSeed({ status: "in_progress" }), + recoveryActionId: "sandbox-recovery", recoveryOwner: null, continuesAutomatically, + sourceRun: { id: "failed-run", status: "failed", errorCode }, + }); + const text = `${notice.body} ${JSON.stringify(notice.metadata)}`; + expect(text).not.toMatch(/inspect|repair|check /i); + if (continuesAutomatically) expect(text).toContain("tries again automatically"); + else { + expect(text).not.toContain("tries again automatically"); + expect(notice.body).toContain("Press Retry to start it again."); + } + } + } }); it("names an environment plugin that was not running after its bounded retries", () => { const notice = buildStrandedRecoveryEscalationNotice({ @@ -146,7 +164,7 @@ describe("buildStrandedRecoveryEscalationNotice", () => { }); expect(notice.presentation.title).toBe("Environment plugin was not running"); expect(notice.body).toContain("before the agent started"); - expect(JSON.stringify(notice.metadata)).toContain("worker status"); + expect(JSON.stringify(notice.metadata)).toContain("Press Retry once it is running again"); }); it("asks for Retry, not a board decision, when the environment provider stayed unavailable", () => { const notice = buildStrandedRecoveryEscalationNotice({ @@ -159,7 +177,7 @@ describe("buildStrandedRecoveryEscalationNotice", () => { const metadata = JSON.stringify(notice.metadata); expect(notice.presentation.title).toBe("Environment provider was temporarily unavailable"); expect(metadata).toContain("Retry when ready"); - expect(metadata).not.toContain("Board decision required"); + expect(metadata).not.toContain("Waiting for you"); }); const actionId = "6a2f8e64-6f5e-4b58-b7fd-111111111111"; const owner = { id: "9b1c2d3e-4f50-4a61-8b72-222222222222", name: "CTO" }; @@ -222,7 +240,7 @@ describe("buildStrandedRecoveryEscalationNotice", () => { const ownerRow = allRows(notice.metadata).find((row) => row.label === "Recovery owner"); expect(ownerRow?.type).toBe("key_value"); - expect(ownerRow?.value).toBe("Board decision required"); + expect(ownerRow?.value).toBe("Waiting for you"); }); it("derives title from the recovery cause and body from the plain-comment fallback", () => { diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index 1367cd2e7e3..c0185da6783 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -11,8 +11,11 @@ import { /** The saved version is unchanged by a failed restore; the run names the failed step. */ export const REMOTE_WORKSPACE_RESTORE_NEXT_ACTION = - "Open the failed run to see which restore step failed. The saved workspace version was kept. " - + "If the sandbox folder is already occupied, inspect and move it aside, then retry the task."; + "The saved work could not be put back into the sandbox, so no agent work started. The saved version was kept. " + + "Press Retry to start again from it."; + +export const RETRY_FROM_SAVED_WORK_NEXT_ACTION = + "This attempt could not be finished. Press Retry to start it again from the saved work."; // Short human-readable body plus the presentation header for one recovery // family. The escalation path merges in the metadata rows only it knows @@ -31,8 +34,7 @@ export type StrandedRecoveryEscalationNotice = { }; export const DEFAULT_STRANDED_RECOVERY_NOTICE_BODY = - "Paperclip could not restore a live execution path for this issue automatically. " + - "Moving it to `blocked` so it is visible for intervention."; + "This attempt could not be finished automatically. Press Retry to start it again from the saved work."; const DEFAULT_STRANDED_RECOVERY_NOTICE_TITLE = "Automatic recovery blocked"; @@ -53,17 +55,26 @@ const STRANDED_RECOVERY_NOTICE_TITLES_BY_RUN_ERROR_CODE: Record acpx_auth_required: "Error: agent login required", }; +const TEMPORARY_SETUP_NEXT_ACTION = "Setting up the workspace took too long, so no agent work started. " + + "Paperclip tries again automatically."; + +const AUTOMATIC_RETRY_SENTENCE = "Paperclip tries again automatically."; +const MANUAL_RETRY_SENTENCE = "Press Retry to start it again."; + +const TEMPORARY_SETUP_CODES = new Set(["remote_workspace_prepare_timeout", "reusable_sandbox_resume_failed"]); + const WORKSPACE_SCAN_NOTICES: Record = { - workspace_git_scan_timeout: { title: "Workspace scan timed out", nextAction: "Check repository access and server load, then retry the task." }, - remote_workspace_prepare_timeout: { title: "Workspace setup timed out", nextAction: "Inspect the failed run and check repository access and sandbox load before retrying the task." }, + workspace_git_scan_timeout: { title: "Workspace scan timed out", nextAction: "Reading the task's files took too long, so no agent work started. Press Retry to start it again." }, + remote_workspace_prepare_timeout: { title: "Workspace setup timed out", nextAction: TEMPORARY_SETUP_NEXT_ACTION }, remote_workspace_restore_failed: { title: "Saved workspace could not be restored", nextAction: REMOTE_WORKSPACE_RESTORE_NEXT_ACTION }, - plugin_worker_unavailable: { title: "Environment plugin was not running", nextAction: "Check the environment plugin's worker status, then retry the task once it is running." }, + plugin_worker_unavailable: { title: "Environment plugin was not running", nextAction: "The plugin that starts this agent's sandbox was not running, so no agent work started. Press Retry once it is running again." }, environment_provider_unavailable: { title: "Environment provider was temporarily unavailable", nextAction: "Paperclip retried automatically before the agent started, but the provider was still unavailable. No agent work started. Retry the task once the provider is available." }, - reusable_sandbox_resume_failed: { title: "Saved sandbox could not be resumed", nextAction: "Inspect the failed run and the sandbox provider; the saved sandbox was kept. Retry the task once the provider can start it." }, - workspace_git_scan_saturated: { title: "Workspace scan queue is full", nextAction: "Check server load and the workspace scan queue, then retry the task." }, - workspace_git_scan_output_limit: { title: "Workspace scan exceeded its limit", nextAction: "Check the repository size and workspace scan output limit before retrying the task." }, - workspace_git_scan_failed: { title: "Workspace scan failed", nextAction: "Inspect the failed run and check repository access and integrity before retrying the task." }, - workspace_git_scan_cancelled: { title: "Workspace scan was cancelled", nextAction: "Inspect why workspace preparation was cancelled before retrying the task." }, + reusable_sandbox_resume_failed: { title: "Saved sandbox could not be resumed", nextAction: "The sandbox service did not start the saved sandbox in time. " + + "The saved sandbox and its files were kept, and no agent work started. Paperclip tries again automatically." }, + workspace_git_scan_saturated: { title: "Workspace scan queue is full", nextAction: "The server was too busy to read the task's files, so no agent work started. Press Retry to start it again." }, + workspace_git_scan_output_limit: { title: "Workspace scan exceeded its limit", nextAction: "The task's files are larger than Paperclip can read at once, so no agent work started. Press Retry to start it again." }, + workspace_git_scan_failed: { title: "Workspace scan failed", nextAction: "The task's files could not be read, so no agent work started. Press Retry to start it again." }, + workspace_git_scan_cancelled: { title: "Workspace scan was cancelled", nextAction: "Preparing the task's files was stopped, so no agent work started. Press Retry to start it again." }, }; const SETUP_FAILURE_NOTICES: Record = { @@ -94,8 +105,8 @@ export function buildImmediateExecutionPathRecoveryNoticeSeed(input: { export function buildWorkspaceValidationRecoveryNoticeSeed(): StrandedRecoveryNoticeSeed { return { body: - "Paperclip stopped before launching the local adapter because the issue workspace failed validation. " + - "Moving it to `blocked` so the workspace link, cwd, or git checkout can be repaired before resuming.", + "The task's workspace could not be used, so the agent did not start. " + + "Press Retry to start it again.", title: "Workspace validation failed", tone: "danger", }; @@ -137,8 +148,8 @@ export function buildConfigurationIncompleteRecoveryNoticeSeed( if (readNonEmptyStringField(configurationIncomplete, "reason") === "workspace_base_ref_unresolved") { return { title: "Workspace base ref unavailable", - body: "Paperclip stopped before the agent started because the configured workspace base ref could not be resolved to a Git commit. The task is blocked until its workspace configuration or repository access is repaired.", - nextAction: "Check that the configured base ref exists and the repository is accessible. Correct the task or project workspace settings, then retry the task.", + body: "The agent did not start because the starting branch set for this task's workspace was not found in the repository.", + nextAction: "Set a starting branch that exists in the repository, then press Retry.", tone: "danger", }; } @@ -147,7 +158,7 @@ export function buildConfigurationIncompleteRecoveryNoticeSeed( return { title: "Shared workspace still held", body: "Paperclip stopped before the agent started because another run still holds this task's shared workspace but has produced no output for more than an hour, and this execution environment cannot place concurrent agents in one physical shared folder.", - nextAction: "Check whether the other run is still working and stop it if it is stuck, or select an isolated task workspace, then retry the task.", + nextAction: "Stop the other run on this task, or choose a separate workspace for this task, then press Retry.", tone: "danger", }; } @@ -215,8 +226,8 @@ export function buildConfigurationIncompleteRecoveryNoticeSeed( } return { body: - "Paperclip stopped before dispatching the adapter because required secret/env bindings are missing. " + - "Moving it to `blocked` so an operator can bind the missing secret(s) before resuming.", + "The agent did not start because a secret it needs is not set. " + + "Add the secret in the agent's settings, then press Retry.", title: "Configuration incomplete", tone: "danger", }; @@ -225,9 +236,8 @@ export function buildConfigurationIncompleteRecoveryNoticeSeed( export function buildExecutionReviewParticipantRecoveryNoticeSeed(): StrandedRecoveryNoticeSeed { return { body: - "Paperclip retried the pending execution-review participant once, but the review stage still has no " + - "completed decision or live reviewer run. Moving it to `blocked` so the board can inspect the evidence, repair the " + - "reviewer runtime, restore the review stage, or record an intentional manual resolution.", + "The reviewer for this task did not finish the review, even after Paperclip asked again. " + + "Choose a reviewer, or press Retry to ask the same reviewer again.", title: "Review recovery stalled", tone: "danger", }; @@ -236,10 +246,8 @@ export function buildExecutionReviewParticipantRecoveryNoticeSeed(): StrandedRec export function buildExecutionReviewParticipantUnavailableNoticeSeed(): StrandedRecoveryNoticeSeed { return { body: - "Paperclip cannot continue the pending execution-review participant because the participant is not " + - "invokable and the review stage has no completed decision or live reviewer run. Moving it to `blocked` " + - "so the board can inspect the evidence, repair the reviewer runtime, restore the review stage, or record an " + - "intentional manual resolution.", + "The reviewer for this task cannot take work right now, so the review could not continue. " + + "Choose another reviewer, or press Retry once the reviewer is available.", title: "Review recovery stalled", tone: "danger", }; @@ -253,6 +261,7 @@ export function buildStrandedRecoveryEscalationNotice(input: { fallbackBody?: string | null; recoveryCause?: string | null; recoveryActionId: string; + continuesAutomatically?: boolean; recoveryOwner: { id: string; name: string | null } | null | undefined; sourceRun: { id: string; @@ -262,10 +271,16 @@ export function buildStrandedRecoveryEscalationNotice(input: { errorSummary?: string | null; } | null | undefined; }): StrandedRecoveryEscalationNotice { - const workspaceScan = WORKSPACE_SCAN_NOTICES[input.sourceRun?.errorCode ?? ""]; + const scanNotice = WORKSPACE_SCAN_NOTICES[input.sourceRun?.errorCode ?? ""]; + const retriesItself = TEMPORARY_SETUP_CODES.has(input.sourceRun?.errorCode ?? "") && input.continuesAutomatically === true; + const workspaceScan = scanNotice && TEMPORARY_SETUP_CODES.has(input.sourceRun?.errorCode ?? "") && !retriesItself + ? { ...scanNotice, nextAction: scanNotice.nextAction.replace(AUTOMATIC_RETRY_SENTENCE, MANUAL_RETRY_SENTENCE) } + : scanNotice; const seed = workspaceScan ? { ...workspaceScan, - body: `Paperclip could not prepare the workspace before the agent started. Automatic recovery could not continue. ${workspaceScan.nextAction}`, + body: retriesItself + ? `Paperclip could not prepare the workspace before the agent started. ${workspaceScan.nextAction}` + : `Paperclip could not prepare the workspace before the agent started. Automatic recovery could not continue. ${workspaceScan.nextAction}`, tone: "danger" as const, } : SETUP_FAILURE_NOTICES[input.sourceRun?.errorCode ?? ""] ?? input.seed; const fallbackBody = input.fallbackBody?.trim(); @@ -283,13 +298,13 @@ export function buildStrandedRecoveryEscalationNotice(input: { : keyValueRow( "Recovery owner", // A pre-dispatch provider outage needs no board decision, only Retry. - input.recoveryCause === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE ? "Retry when ready" : "Board decision required", + input.recoveryCause === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE ? "Retry when ready" : "Waiting for you", ), keyValueRow( "Next action", seed?.nextAction ?? (input.recoveryOwner - ? "The recovery owner should either restore a live execution path or record the manual resolution on the source issue" - : "Inspect the evidence, then retry the original owner, explicitly reassign, repair the execution path, or record an intentional resolution"), + ? "The assigned agent continues the task from the saved work" + : RETRY_FROM_SAVED_WORK_NEXT_ACTION), ), ]; diff --git a/server/src/services/recovery/successful-run-handoff.ts b/server/src/services/recovery/successful-run-handoff.ts index 5c199f73cf4..b6971ceedf8 100644 --- a/server/src/services/recovery/successful-run-handoff.ts +++ b/server/src/services/recovery/successful-run-handoff.ts @@ -233,9 +233,9 @@ export function buildSuccessfulRunHandoffExhaustedNotice(input: { : issueLinkRow("Recovery issue", input.recoveryIssue), input.recoveryOwner ? agentLinkRow("Recovery owner", input.recoveryOwner) - : keyValueRow("Recovery owner", "Board decision required"), + : keyValueRow("Recovery owner", "Waiting for you"), agentLinkRow("Source assignee", input.sourceAssignee), - keyValueRow("Suggested action", "inspect the evidence, then retry the original owner, explicitly reassign, or record a valid issue disposition"), + keyValueRow("Suggested action", "Mark the task done if it is, or press Retry to let the agent continue from the saved work"), ], }, { diff --git a/server/src/services/task-retry.test.ts b/server/src/services/task-retry.test.ts index dacfd42bda1..6bd15aece7a 100644 --- a/server/src/services/task-retry.test.ts +++ b/server/src/services/task-retry.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { agents, agentTaskSessions, agentWakeupRequests, chatActions, chatConversations, chatEndpoints, companies, budgetPolicies, costEvents, createDb, type Db, environmentLeases, environments, executionWorkspaces, executionWorkspaceSnapshots, workspaceOperations, heartbeatRunEvents, heartbeatRuns, issueComments, issues, approvals, issueApprovals, issueRecoveryActions, nativeRunFinalizations, toolApplications, toolConnections, toolInvocations, applyPendingMigrations } from "@paperclipai/db"; +import { activityLog, agents, agentTaskSessions, agentWakeupRequests, chatActions, chatConversations, chatEndpoints, companies, budgetPolicies, costEvents, createDb, type Db, environmentLeases, environments, executionWorkspaces, executionWorkspaceSnapshots, workspaceOperations, heartbeatRunEvents, heartbeatRuns, issueComments, issueRelations, issues, approvals, issueApprovals, issueRecoveryActions, nativeRunFinalizations, toolApplications, toolConnections, toolInvocations, applyPendingMigrations } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { priorRecoveryApiReceipt, unresolvedRecoveryToolInvocation } from "./recovery-effect-receipts.js"; import { requestTaskRetry } from "./task-retry.js"; @@ -740,6 +740,33 @@ describe("durable task Retry admission", () => { await expect(dispatchRun(input, queued[0])).resolves.toMatchObject({ interruptedRunId: refused.id }); }); + it("lifts the failure's block once the Retry's run is queued, and keeps a block made for another reason", async () => { + for (const reason of ["failure", "declared", "dependency", "newer", "person"] as const) { + const { input, source, refused, hold } = await seedRefusedRetry(); + if (reason === "newer") await db.update(issueRecoveryActions).set({ evidence: { ...hold.evidence, + nativeFailureBlock: { runId: source.id, statusVersion: 2 } } }).where(eq(issueRecoveryActions.id, hold.id)); + await db.update(issues).set({ blockedTransitionAt: new Date(), statusVersion: 3, + ...(reason === "declared" ? { unblockDescriptor: { owner: { userId: "someone" }, action: "Add the key" } } : {}), + }).where(eq(issues.id, input.issueId)); + if (reason === "person") await db.insert(activityLog).values({ companyId: input.companyId, actorType: "user", actorId: "board-user", + action: "issue.updated", entityType: "issue", entityId: input.issueId, details: { status: "blocked", previousStatus: "todo" } }); + if (reason === "dependency") { + const [blocker] = await db.insert(issues).values({ companyId: input.companyId, title: "Blocker", status: "todo" }).returning(); + await db.insert(issueRelations).values({ companyId: input.companyId, issueId: blocker.id, relatedIssueId: input.issueId, type: "blocks" }); + } + await db.update(agents).set({ runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } }).where(eq(agents.id, source.agentId)); + await db.insert(heartbeatRuns).values({ companyId: input.companyId, agentId: source.agentId, status: "running" }); + const receipt = await requestTaskRetry(input); + await heartbeatService(db).resumeTaskRetry(input.companyId, receipt.requestId!); + const queued = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.status, "queued"))); + const [task] = await db.select().from(issues).where(eq(issues.id, input.issueId)); + if (reason === "failure") { + expect(queued[0]?.contextSnapshot).toMatchObject({ explicitUserContinuation: { previousRunId: refused.id } }); + expect(task).toMatchObject({ status: "todo", blockedTransitionAt: null }); + } else expect(task.status, reason).toBe("blocked"); + } + }); + it("serializes concurrent clicks and remembers each key after completion", async () => { const input = await seed(); const second = { ...input, idempotencyKey: randomUUID() };