diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index f40b0d7e086..19eec94b7dc 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -8132,8 +8132,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); const comments = await selectRecoveryCommentsForIssue(issueId); expect(comments).toHaveLength(1); - expect(comments[0]?.body).toContain("bounded retry budget"); - expect(comments[0]?.body).toContain("no live execution path"); + expect(comments[0]?.body).toContain("Paperclip tried to continue this task again, but it still could not continue"); + expect(comments[0]?.body).toContain("Press Retry"); expect(comments[0]?.authorType).toBe("system"); expect(comments[0]?.presentation).toMatchObject({ kind: "system_notice", @@ -8229,7 +8229,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(sourceIssue?.status).toBe("blocked"); expect(sourceIssue?.assigneeAgentId).toBe(agentId); const comments = await selectRecoveryCommentsForIssue(issueId); - expect(comments[0]?.body).toContain("bounded retry budget"); + expect(comments[0]?.body).toContain("Paperclip tried to start this task again, but it still could not start. Press Retry"); }); it("escalates an exhausted successful handoff run that still leaves no disposition", async () => { @@ -12370,7 +12370,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const comments = await selectRecoveryCommentsForIssue(issueId); expect(comments).toHaveLength(1); - expect(comments[0]?.body).toContain("retried dispatch"); + expect(comments[0]?.body).toContain("tried to start this task again"); expect(comments[0]?.body).not.toContain("sk-test-recovery-secret"); expect(JSON.stringify(comments[0]?.metadata)).not.toContain( "sk-test-recovery-secret", @@ -12474,13 +12474,14 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const comments = await selectRecoveryCommentsForIssue(issueId); expect(comments).toHaveLength(1); expect(comments[0]?.body).toContain( - "stopped automatic stranded-work recovery", + "Paperclip stopped trying to recover this task automatically.", ); expect(comments[0]?.body).toContain( - "recovery issues do not create nested `stranded_issue_recovery` issues", + "This task was created to recover another task, so Paperclip does not create another recovery task for it.", ); + expect(comments[0]?.body).not.toMatch(/stranded_issue_recovery|Retry reason|assignment_recovery/); expect(comments[0]?.body).toContain( - `Recovery issue: [${recoveryIssues[0]?.identifier}]`, + `Recovery task: [${recoveryIssues[0]?.identifier}]`, ); expect(comments[0]?.body).toContain("Next action:"); expect(comments[0]?.presentation).toMatchObject({ @@ -12860,7 +12861,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const comments = await selectRecoveryCommentsForIssue(issueId); expect(comments).toHaveLength(1); - expect(comments[0]?.body).toContain("retried continuation"); + expect(comments[0]?.body).toContain("tried to continue this task again"); expect(comments[0]?.presentation).toMatchObject({ kind: "system_notice", tone: "danger", @@ -13014,8 +13015,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const comments = await selectRecoveryCommentsForIssue(issueId); expect(comments).toHaveLength(1); - expect(comments[0]?.body).toContain("retried continuation"); - expect(comments[0]?.body).toContain("3× attempts"); + expect(comments[0]?.body).toContain("tried to continue this task again"); + expect(comments[0]?.body).toContain("after 3 tries"); expect(commentMetadataRows(comments[0])).toContainEqual({ type: "key_value", label: "Failure code", @@ -13079,8 +13080,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const comments = await selectRecoveryCommentsForIssue(issueId); expect(comments).toHaveLength(1); - expect(comments[0]?.body).toContain("non-retryable failure"); - expect(comments[0]?.body).toContain("`budget_blocked`"); + expect(comments[0]?.body).toContain("retrying alone would not fix"); + expect(comments[0]?.body).not.toContain("budget_blocked"); const followupRuns = await selectRecoveryRunsByAgent(agentId); const continuationRetryRun = followupRuns.find((row) => { @@ -14931,13 +14932,13 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const comments = await selectRecoveryCommentsForIssue(issueId); expect(comments).toHaveLength(1); expect(comments[0]?.body).toContain( - "stopped automatic stranded-work recovery", + "Paperclip stopped trying to recover this task automatically.", ); expect(comments[0]?.body).toContain( "The failure details are kept with the run", ); expect(comments[0]?.body).toContain( - "recovery issues do not create nested `stranded_issue_recovery` issues", + "This task was created to recover another task, so Paperclip does not create another recovery task for it.", ); await expect( sourceBlockerIssueIds(companyId, sourceIssueId), diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index b78c1338523..96988c8a8a1 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -15,6 +15,7 @@ import { readClaimedNativeStopRecovery } from "../../../services/native-runtime/ import { hasConversationContinuationPolicy } from "../../../services/conversation-continuation.js"; import { recordedUserStop, userStopBoundaryAt } from "../../../services/run-cancellation.js"; import { getExecutionBlocker } from "../../../services/execution-blocker.js"; +import { continueFreshInTransaction, recordFreshContinuationCheck } from "../../../services/fresh-continuation-fallback.js"; import { validateReconstructedWorkspaceDispatch } from "../../../services/native-runtime/native-workspace-reconstruction.js"; import { getNativeReviewAssignment } from "../../../services/native-runtime/native-review-participant.js"; import { and, asc, eq, gte, inArray, isNull, lte, or, sql } from "drizzle-orm"; @@ -1380,6 +1381,19 @@ export function createPostgresRunDispatchAdapter( return { outcome: "deferred" as const, postCommitEffects: [statusEffect(waiting, run.status)] }; } + async function settleOlderRunHold(tx: Db, run: HeartbeatRun, issueId: string, + blocker: NonNullable>>, now: Date) { + if (!blocker.recoveryActionId || !blocker.runId || blocker.runId === run.id) return; + const [issue] = await tx.select({ conversationAgentId: issues.conversationAgentId }).from(issues) + .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))); + if (!issue || issue.conversationAgentId) return; + const fresh = await continueFreshInTransaction(tx, { companyId: run.companyId, runId: blocker.runId, + trigger: "continuation_wake", delivery: "record", now }); + if (fresh.kind === "exact_recovery_pending") + await recordFreshContinuationCheck(tx, { companyId: run.companyId, actionId: blocker.recoveryActionId }, fresh, now); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition); + } + async function cancelStaleQueuedRun( input: CancelStaleQueuedRunInput, ): Promise { @@ -1404,7 +1418,9 @@ export function createPostgresRunDispatchAdapter( const deferred = await deferRetryForCleanupInTx(tx, run, issueId, input.now, executionBlocker); if (deferred) return deferred; } - return cancelStaleRunInTx(tx, run, issueId, decision, input.expectedStatus, input.now); + const cancelled = await cancelStaleRunInTx(tx, run, issueId, decision, input.expectedStatus, input.now); + if (cancelled.outcome === "cancelled" && executionBlocker) await settleOlderRunHold(tx, run, issueId, executionBlocker, input.now); + return cancelled; }; return withIssueThenRunLocks( diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index b543a13108c..d7cf5b54fe0 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -12,6 +12,7 @@ import { conversationRecoveryActionPredicate, getConversationOwnershipBlocker } import { isInterruptedNativePreparationCandidate } from "./cancelled-native-startup.js"; import { persistActivity } from "./activity-log.js"; import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; +import { foldWaitingRetriesIntoContinuation } from "./fresh-continuation-retry-fold.js"; import { logger } from "../middleware/logger.js"; import { and, asc, eq, exists, inArray, isNull, not, or, sql } from "drizzle-orm"; import { @@ -424,6 +425,7 @@ export async function deliverReconciledExecutions( where queued_recovery.company_id = ${issueRecoveryActions.companyId} and queued_recovery.agent_id = ${issueRecoveryActions.returnOwnerAgentId} and queued_recovery.status = 'queued' + and queued_recovery.context_snapshot->>'recoveryActionId' is distinct from ${issueRecoveryActions.id}::text and queued_recovery.context_snapshot->>'wakeReason' in (${sql.join(recoveryReasons.map(reason => sql`${reason}`), sql`, `)})))`, ), ) @@ -521,7 +523,7 @@ export async function deliverReconciledExecutions( // recorded already continues the task, so it is never a second run. const fresh = action.evidence.freshContinuation as { recordedAt?: unknown } | undefined; if (fresh && typeof fresh.recordedAt === "string") { - const [newer] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + const [newer] = await db.select({ id: heartbeatRuns.id, contextSnapshot: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, action.companyId), sql`${heartbeatRuns.id}::text <> ${decision.runId}`, or(eq(heartbeatRuns.nativeIssueId, task.id), eq(heartbeatRuns.issueId, task.id), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${task.id}::text`), @@ -532,6 +534,12 @@ export async function deliverReconciledExecutions( .where(pendingDecision); await settleCoalescedFreshContinuationHolds(db, { companyId: action.companyId, primaryId: action.id, delivery: "delivered", continuationRunId: newer.id }); + if (newer.contextSnapshot?.freshContinuationIsolation === action.id) + await db.transaction(async tx => { + await tx.execute(sql`select id from issues where company_id = ${action.companyId} and id = ${task.id} for update`); + await foldWaitingRetriesIntoContinuation(tx as unknown as Db, { companyId: action.companyId, issueId: task.id, runId: newer.id, + sourceRunId: decision.runId }); + }); continue; } } @@ -592,6 +600,9 @@ export async function deliverReconciledExecutions( .where(pendingDecision); await settleCoalescedFreshContinuationHolds(tx as unknown as Db, { companyId: action.companyId, primaryId: action.id, delivery: "delivered", continuationRunId: run.id }); + if ((action.evidence.freshContinuation as { isolation?: unknown } | undefined)?.isolation === "fresh_isolated_workspace") + await foldWaitingRetriesIntoContinuation(tx as unknown as Db, { companyId: action.companyId, issueId: task.id, runId: run.id, + sourceRunId: decision.runId }); }); } catch { if (action.returnOwnerAgentId) seenAgents.add(action.returnOwnerAgentId); diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index a8fe99ae905..390b322b19f 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -24,6 +24,10 @@ import { agentRunWritesRevoked, assertAgentRunWriteAllowed } from "../agent-run- import { loadAgentRunAuthentication } from "../middleware/agent-run-authentication.js"; import { SERVER_RESTART_RETRY_MAX_ATTEMPTS, SERVER_RESTART_RETRY_WINDOW_MS } from "./execution-recovery-attempt.js"; import { FRESH_CONTINUATION_WINDOW_MS } from "./fresh-continuation-fallback.js"; +import { createPostgresRunDispatchAdapter } from "../modules/run-dispatch/adapters/postgres.js"; +import { createSystemRecovery } from "./system-recovery-state.js"; +import { requestTaskRetry } from "./task-retry.js"; +import { terminalizeLegacyExecution } from "./legacy-execution-recovery.js"; const support = await getEmbeddedPostgresTestSupport(); const suite = support.supported ? describe : describe.skip; @@ -316,6 +320,93 @@ suite("fresh-continuation fallback", () => { expect((await pendingRecord(f.issueId))!.evidence.continuationDelivery).toBe("delivered"); }, 60_000); + it("runs a person's Retry pressed before delivery as the recorded isolated continuation, never in the shared directory", async () => { + const f = await unfinishedRun({ runtime: "legacy", errorCode: "controller_lease_lost", error: "Legacy controller lease lost" }); + await db.update(heartbeatRuns).set({ processPid: 4242, processHostHash: "replaced-server", processStartedAt: new Date(Date.now() - 600_000) }) + .where(eq(heartbeatRuns.id, f.runId)); + await db.update(issues).set({ projectId: null, projectWorkspaceId: null, executionWorkspaceId: null }).where(eq(issues.id, f.issueId)); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })) + .toMatchObject({ kind: "continued", isolation: "fresh_isolated_workspace", delivery: "recorded" }); + await occupySlot(); + const taskRuns = async () => db.select().from(heartbeatRuns).where(and( + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`, sql`${heartbeatRuns.id} <> ${f.runId}`)); + const retry = await requestTaskRetry({ db, companyId, issueId: f.issueId, actorId: userId, observedRunId: f.runId, idempotencyKey: randomUUID() }) + .catch((error: unknown) => ({ error: String(error) })); + await deliver(); + expect(retry).toMatchObject({ accepted: true }); + const retryWake = async () => (await db.select().from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, (retry as { requestId: string }).requestId)))[0]; + const runs = await taskRuns(); + expect(runs).toHaveLength(1); + expect(await retryWake()).toMatchObject({ status: "coalesced", runId: runs[0]!.id }); + expect(runs[0]!.contextSnapshot).toMatchObject({ freshContinuationIsolation: f.actionId }); + expect(await readFreshWorkspaceReadiness(db, runs[0]!)).toMatchObject({ mode: "fresh_isolated_workspace", retryIntentId: f.actionId }); + const endedAt = new Date(); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: endedAt }) + .where(and(eq(heartbeatRuns.status, "running"), sql`${heartbeatRuns.contextSnapshot}->>'issueId' <> ${f.issueId}`)); + await heartbeatService(db).cancelRun(runs[0]!.id); + await heartbeatService(db).resumeQueuedRuns(); + await deliver(); + const later = (await taskRuns()).filter(run => run.id !== runs[0]!.id); + expect(later.filter(run => run.wakeupRequestId === (retry as { requestId: string }).requestId + || run.contextSnapshot?.wakeReason === "task_retry")).toEqual([]); + expect(await retryWake()).toMatchObject({ status: "coalesced", runId: runs[0]!.id }); + }, 60_000); + + it("consumes a person's Retry exactly once when the server stops right after admitting the isolated continuation", async () => { + const f = await unfinishedRun({ runtime: "legacy", errorCode: "controller_lease_lost", error: "Legacy controller lease lost" }); + await db.update(heartbeatRuns).set({ processPid: 4242, processHostHash: "replaced-server", processStartedAt: new Date(Date.now() - 600_000) }) + .where(eq(heartbeatRuns.id, f.runId)); + await db.update(issues).set({ projectId: null, projectWorkspaceId: null, executionWorkspaceId: null }).where(eq(issues.id, f.issueId)); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })) + .toMatchObject({ kind: "continued", isolation: "fresh_isolated_workspace", delivery: "recorded" }); + await occupySlot(); + const retry = await requestTaskRetry({ db, companyId, issueId: f.issueId, actorId: userId, observedRunId: f.runId, idempotencyKey: randomUUID() }); + expect(retry).toMatchObject({ accepted: true }); + const retryWake = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.requestId!)))[0]; + const taskRuns = async () => db.select().from(heartbeatRuns).where(and( + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`, sql`${heartbeatRuns.id} <> ${f.runId}`)); + const admitted = await heartbeatService(db).wakeup(agentId, { source: "automation", triggerDetail: "system", + reason: "issue_recovery_action_restored", idempotencyKey: `execution-reconciliation:${f.actionId}`, + payload: { issueId: f.issueId, recoveryActionId: f.actionId }, requestedByActorType: "system", requestedByActorId: "execution-recovery", + contextSnapshot: { issueId: f.issueId, taskId: f.issueId, recoveryActionId: f.actionId, previousRunId: f.runId, retryOfRunId: f.runId, + forceFreshSession: true, wakeReason: "issue_recovery_action_restored", source: "execution.reconciled", + freshContinuationIsolation: f.actionId } }); + expect(admitted).toBeTruthy(); + expect(await retryWake()).toMatchObject({ status: "coalesced", runId: admitted!.id }); + await db.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null }) + .where(eq(agentWakeupRequests.id, retry.requestId!)); + await deliver(); + expect((await taskRuns()).map(run => run.id)).toEqual([admitted!.id]); + expect(await retryWake()).toMatchObject({ status: "coalesced", runId: admitted!.id }); + expect((await pendingRecord(f.issueId))!.evidence).toMatchObject({ continuationDelivery: "delivered", continuationRunId: admitted!.id }); + }, 60_000); + + it("never folds a Retry for the admitted continuation's own failure into that ended continuation", async () => { + const f = await unfinishedRun({ runtime: "legacy", errorCode: "controller_lease_lost", error: "Legacy controller lease lost" }); + await db.update(heartbeatRuns).set({ processPid: 4242, processHostHash: "replaced-server", processStartedAt: new Date(Date.now() - 600_000) }) + .where(eq(heartbeatRuns.id, f.runId)); + await db.update(issues).set({ projectId: null, projectWorkspaceId: null, executionWorkspaceId: null }).where(eq(issues.id, f.issueId)); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "recovery_refused" })) + .toMatchObject({ kind: "continued", isolation: "fresh_isolated_workspace", delivery: "recorded" }); + const admitted = await heartbeatService(db).wakeup(agentId, { source: "automation", triggerDetail: "system", + reason: "issue_recovery_action_restored", idempotencyKey: `execution-reconciliation:${f.actionId}`, + payload: { issueId: f.issueId, recoveryActionId: f.actionId }, requestedByActorType: "system", requestedByActorId: "execution-recovery", + contextSnapshot: { issueId: f.issueId, taskId: f.issueId, recoveryActionId: f.actionId, previousRunId: f.runId, retryOfRunId: f.runId, + forceFreshSession: true, wakeReason: "issue_recovery_action_restored", source: "execution.reconciled", + freshContinuationIsolation: f.actionId } }); + await heartbeatService(db).cancelRun(admitted!.id); + const retry = await requestTaskRetry({ db, companyId, issueId: f.issueId, actorId: userId, observedRunId: admitted!.id, idempotencyKey: randomUUID() }); + expect(retry).toMatchObject({ accepted: true }); + await deliver(); + await heartbeatService(db).resumeQueuedRuns(); + await vi.waitFor(async () => { + const [retryWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.requestId!)); + expect(retryWake!.runId, JSON.stringify(retryWake)).toBeTruthy(); + expect(retryWake!.runId).not.toBe(admitted!.id); + }, { timeout: 20_000, interval: 200 }); + }, 60_000); + it("re-drives an isolated continuation pending past the bound itself; a repair wake never replaces it", async () => { const f = await unfinishedRun({ runtime: "legacy", errorCode: "controller_lease_lost", error: "Legacy controller lease lost" }); await db.update(heartbeatRuns).set({ processPid: 4242, processHostHash: "replaced-server", processStartedAt: new Date(Date.now() - 600_000) }) @@ -1042,4 +1133,192 @@ suite("fresh-continuation fallback", () => { .toMatchObject({ kind: "already_continued" }); expect(await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests)).toHaveLength(0); }, 60_000); + + async function heldByOlderRun(options: { endedAgoMs?: number; recovery?: "exhausted" | "running"; humanGate?: boolean } = {}) { + const f = await unfinishedRun({ runtime: "legacy", errorCode: "process_lost" }); + const endedAt = new Date(Date.now() - (options.endedAgoMs ?? 60_000)); + await db.update(heartbeatRuns).set({ finishedAt: endedAt, startedAt: new Date(endedAt.getTime() - 600_000), + createdAt: new Date(endedAt.getTime() - 660_000), processPid: 4242, processHostHash: "replaced-server", + processStartedAt: new Date(endedAt.getTime() - 600_000) }).where(eq(heartbeatRuns.id, f.runId)); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: f.issueId, mode: "isolated", + strategyType: "git_worktree", name: "Task workspace", providerType: "local_fs", cwd: "/workspaces/old-task-dir" }).returning(); + await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, f.issueId)); + const recovery = { ...createSystemRecovery(f.runId), state: options.recovery ?? "exhausted", attempts: 3, + failure: { code: "export_interrupted", retryClass: "retry_transient", evidenceFingerprint: "export" } }; + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId!)); + await db.update(issueRecoveryActions).set({ ownerUserId: options.humanGate ? userId : null, + createdAt: endedAt, updatedAt: endedAt, + evidence: { ...hold!.evidence, workspaceRestoreFailure: "restore_unsafe_archive", systemRecovery: recovery } }) + .where(eq(issueRecoveryActions.id, f.actionId!)); + return { ...f, workspaceId: workspace!.id, endedAt }; + } + + async function refusedAtDispatch(f: { issueId: string }, options: { createdAt?: Date; wakeReason?: string } = {}) { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, + runtimeMode: "legacy", status: "queued", createdAt: options.createdAt ?? new Date(), + contextSnapshot: { issueId: f.issueId, wakeReason: options.wakeReason ?? "issue_continuation_needed" } }).returning(); + const outcome = await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId, runId: run!.id, + expectedStatus: "queued", now: options.createdAt ?? new Date() }); + return { run: run!, outcome }; + } + + const holdRow = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!; + const makeDue = (id: string) => db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, + '{freshContinuationCheck,nextCheckAt}', to_jsonb(${Date.now() - 1_000}::bigint))` }).where(eq(issueRecoveryActions.id, id)); + + it("continues a task held by an older run's saved-work hold in a new isolated workspace, after it refused a run at dispatch", async () => { + const f = await heldByOlderRun(); + const refused = await refusedAtDispatch(f); + expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" }); + expect((await holdRow(f.actionId!)).evidence.freshContinuationCheck) + .toMatchObject({ kind: "exact_recovery_pending", firstPendingAt: expect.any(Number) }); + expect(await heldExecutionsAwaitingFreshContinuation(db, new Date(Date.now() + 10 * 60_000), 10)) + .toEqual([expect.objectContaining({ id: f.actionId, runId: f.runId })]); + await waitedFor(f.actionId!, FRESH_CONTINUATION_PENDING_BOUND_MS + 1_000); + await makeDue(f.actionId!); + expect(await continueHeldExecutions(db)).toMatchObject({ continued: 1 }); + const settled = await holdRow(f.actionId!); + expect(settled).toMatchObject({ status: "resolved", ownerType: "system" }); + expect(settled.evidence).toMatchObject({ workspaceRestoreFailure: "restore_unsafe_archive", + systemRecovery: { state: "exhausted" }, + freshContinuation: { sourceRunId: f.runId, isolation: "fresh_isolated_workspace" }, + freshContinuationSettlement: { actorType: "system", actorId: "fresh-continuation", heldFiles: "kept_not_applied", + refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } }); + expect((await issue(f.issueId)).executionWorkspaceId).toBeNull(); + const [old] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.workspaceId)); + expect(old).toMatchObject({ status: "active", cwd: "/workspaces/old-task-dir" }); + const [oldRun] = await loadAgentRunAuthentication(db, { runId: f.runId, companyId, agentId }); + expect(agentRunWritesRevoked(oldRun)).toBe(true); + expect(await getExecutionBlocker(db, companyId, f.issueId)).toBeNull(); + await occupySlot(); + await deliver(); + const [next] = await successors(f.runId); + expect(next!.contextSnapshot).toMatchObject({ freshContinuationIsolation: f.actionId }); + expect(await readFreshWorkspaceReadiness(db, next!)).toMatchObject({ mode: "fresh_isolated_workspace", sourceRunId: f.runId }); + expect(await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId, runId: next!.id, + expectedStatus: "queued", now: new Date() })).not.toMatchObject({ errorCode: "execution_reconciliation_required" }); + }, 60_000); + + it("keeps the wait of a saved-work recovery that is still advancing, even behind a run it refused", async () => { + const f = await heldByOlderRun({ recovery: "running" }); + await refusedAtDispatch(f); + await waitedFor(f.actionId!, FRESH_CONTINUATION_PROGRESS_BOUND_MS + 1_000); + await db.update(issueRecoveryActions).set({ updatedAt: new Date(Date.now() + 1_000), + evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, '{systemRecovery,attempts}', '4'::jsonb)` }) + .where(eq(issueRecoveryActions.id, f.actionId!)); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "held_recovery" })) + .toMatchObject({ kind: "exact_recovery_pending", reason: "saved_work_recovery_running" }); + expect((await issue(f.issueId)).executionWorkspaceId).toBe(f.workspaceId); + await refusedAtDispatch(f, { wakeReason: "task_retry" }); + await db.update(issueRecoveryActions).set({ updatedAt: new Date(Date.now() + 2_000), + evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, '{systemRecovery,attempts}', '5'::jsonb)` }) + .where(eq(issueRecoveryActions.id, f.actionId!)); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "held_recovery" })) + .toMatchObject({ kind: "exact_recovery_pending" }); + }, 60_000); + + it("keeps a hold a person owns blocking, behind any number of refused runs", async () => { + const f = await heldByOlderRun({ humanGate: true }); + await refusedAtDispatch(f, { wakeReason: "task_retry" }); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "held_recovery" })) + .toMatchObject({ kind: "person_owned" }); + expect(await getExecutionBlocker(db, companyId, f.issueId)).toMatchObject({ recoveryActionId: f.actionId }); + expect((await holdRow(f.actionId!)).status).toBe("active"); + }, 60_000); + + it("leaves an old hold for a person's Retry, and that Retry runs isolated (refused run)", async () => { + const f = await heldByOlderRun({ endedAgoMs: 3 * 86_400_000 }); + await refusedAtDispatch(f, { createdAt: new Date(Date.now() - 30 * 3_600_000) }); + expect(await continueHeldExecutions(db)).toMatchObject({ continued: 0 }); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "continuation_wake" })) + .toEqual({ kind: "person_owned", reason: "retry_needed" }); + expect((await holdRow(f.actionId!)).status).toBe("active"); + const retry = await refusedAtDispatch(f, { wakeReason: "task_retry" }); + expect(retry.outcome).toMatchObject({ outcome: "cancelled" }); + expect((await holdRow(f.actionId!)).evidence).toMatchObject({ + freshContinuation: { isolation: "fresh_isolated_workspace" }, + freshContinuationSettlement: { requestedBy: "person_retry", heldFiles: "kept_not_applied" } }); + await occupySlot(); + await deliver(); + const [next] = await successors(f.runId); + expect(await readFreshWorkspaceReadiness(db, next!)).toMatchObject({ mode: "fresh_isolated_workspace" }); + }, 60_000); + + it("leaves an old hold for a person's Retry, and that Retry runs isolated (waiting Retry)", async () => { + const f = await heldByOlderRun({ endedAgoMs: 3 * 86_400_000 }); + const old = await refusedAtDispatch(f, { createdAt: new Date(Date.now() - 30 * 3_600_000) }); + expect(await continueHeldExecutions(db)).toMatchObject({ continued: 0 }); + const receipt = await requestTaskRetry({ db, companyId, issueId: f.issueId, actorId: userId, + idempotencyKey: randomUUID(), observedRunId: old.run.id }); + expect(receipt).toMatchObject({ accepted: true }); + expect(await continueHeldExecutions(db)).toMatchObject({ continued: 1 }); + expect((await holdRow(f.actionId!)).evidence).toMatchObject({ freshContinuation: { isolation: "fresh_isolated_workspace" }, + freshContinuationSettlement: { requestedBy: "person_retry" } }); + await occupySlot(); + await deliver(); + expect(await successors(f.runId)).toHaveLength(1); + const [retryWake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!)); + expect(retryWake!.status).toBe("coalesced"); + }, 60_000); + + async function preparedRemoteRunStoppedBeforeDispatch(options: { endedAgoMs?: number } = {}) { + const issueId = randomUUID(), runId = randomUUID(); + const endedAt = new Date(Date.now() - (options.endedAgoMs ?? 60_000)); + await db.insert(issues).values({ id: issueId, companyId, title: "Ship the release notes", status: "in_progress", assigneeAgentId: agentId }); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issueId, mode: "isolated", + strategyType: "git_worktree", name: "Task workspace", cwd: "/work/remote-task" }).returning(); + await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issueId)); + const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "assignment", issueId, + runtimeMode: "legacy", status: "running", startedAt: new Date(endedAt.getTime() - 120_000), + createdAt: new Date(endedAt.getTime() - 180_000), responsibleUserId: userId, + contextSnapshot: { issueId, executionWorkspaceId: workspace!.id, wakeReason: "issue_assigned" } }).returning(); + const [environment] = await db.insert(environments).values({ name: `sandbox-${runId}`, driver: "sandbox" }).returning(); + const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, executionWorkspaceId: workspace!.id, + heartbeatRunId: runId, environmentId: environment!.id, status: "active", leasePolicy: "ephemeral", provider: "daytona", + providerLeaseId: `sandbox-${runId}`, metadata: { remoteCwd: "/work/remote-task", driver: "sandbox", + workspaceRealization: { authoritativeRoot: "/work/remote-task" } } }).returning(); + await terminalizeLegacyExecution({ db, run: run!, status: "interrupted", patch: { errorCode: "server_shutdown_interrupted", + error: "Interrupted by graceful server shutdown (SIGTERM)", finishedAt: endedAt, + resultJson: { workspaceRestoreFailure: "restore_failed" } } }); + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + return { issueId, runId, actionId: hold!.id, leaseId: lease!.id, workspaceId: workspace!.id }; + } + + it("continues a recent restart-stopped run whose prepared remote workspace could not be restored, in a new isolated workspace", async () => { + const f = await preparedRemoteRunStoppedBeforeDispatch(); + const held = await holdRow(f.actionId); + await settleUnrecoverableExecutions(db); + await waitedFor(f.actionId, FRESH_CONTINUATION_PENDING_BOUND_MS + 1_000).catch(() => undefined); + if ((await holdRow(f.actionId)).evidence.freshContinuationCheck) await makeDue(f.actionId); + await settleUnrecoverableExecutions(db); + const settled = await holdRow(f.actionId); + expect(settled.evidence).toMatchObject({ freshContinuation: { sourceRunId: f.runId, isolation: "fresh_isolated_workspace" }, + workspaceRestoreRecovery: { leaseIds: [f.leaseId] }, freshContinuationSettlement: { heldFiles: "kept_not_applied" } }); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + expect(lease!.leasePolicy).toBe("retain_on_failure"); + expect((await issue(f.issueId)).executionWorkspaceId).toBeNull(); + expect(await getExecutionBlocker(db, companyId, f.issueId)).toBeNull(); + await occupySlot(); + await deliver(); + const [next] = await successors(f.runId); + expect(await readFreshWorkspaceReadiness(db, next!)).toMatchObject({ mode: "fresh_isolated_workspace", sourceRunId: f.runId }); + expect(held.nextAction).not.toMatch(/stop receipt|recover the missing files|workspaceRepairEvidence|allocation|repair|inspect|_/i); + expect(held.nextAction).toContain("Retry"); + }, 60_000); + + it("leaves an older restart-stopped remote restore failure for a person's Retry, which then runs isolated", async () => { + const f = await preparedRemoteRunStoppedBeforeDispatch({ endedAgoMs: 30 * 3_600_000 }); + await settleUnrecoverableExecutions(db); + await settleUnrecoverableExecutions(db); + const kept = await holdRow(f.actionId); + expect(kept.evidence.freshContinuation).toBeUndefined(); + expect(await getExecutionBlocker(db, companyId, f.issueId)).toMatchObject({ recoveryActionId: f.actionId }); + await refusedAtDispatch(f, { wakeReason: "task_retry" }); + expect((await holdRow(f.actionId)).evidence).toMatchObject({ freshContinuation: { isolation: "fresh_isolated_workspace" }, + freshContinuationSettlement: { requestedBy: "person_retry" } }); + await occupySlot(); + await deliver(); + const [next] = await successors(f.runId); + expect(await readFreshWorkspaceReadiness(db, next!)).toMatchObject({ mode: "fresh_isolated_workspace" }); + }, 60_000); }); diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index b164039d5bc..2c5bcac3257 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -1,4 +1,4 @@ -import { and, asc, desc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; +import { and, asc, desc, eq, gt, inArray, isNull, or, sql, type SQL } from "drizzle-orm"; import { agentWakeupRequests, agents, approvals, companies, documentRevisions, documents, executionWorkspaces, heartbeatRuns, issueApprovals, issueComments, issueRecoveryActions, issueThreadInteractions, issues, nativeRunFinalizations, toolInvocations, type Db, @@ -441,6 +441,22 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< // resolved history is never rewritten. const ownedHolds = holds.filter(isOwnedReconciliationHold); + const [refused] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.id} <> ${run.id}`, + or(eq(heartbeatRuns.nativeIssueId, issue.id), eq(heartbeatRuns.issueId, issue.id), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`), + sql`(${heartbeatRuns.createdAt}, ${heartbeatRuns.id}) > (${run.createdAt.toISOString()}::timestamptz, ${run.id}::uuid)`, + eq(heartbeatRuns.status, "cancelled"), isNull(heartbeatRuns.startedAt), eq(heartbeatRuns.errorCode, "execution_reconciliation_required"))) + .orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); + let personAsked = false; + if (refused) { + if (!holds.length) return { kind: "successor_exists" }; + const rule = interruptionEligibility({ now, companyId: sql`${companyId}`, issueId: sql`${issue.id}`, sourceRunId: sql`${run.id}` }); + const [decision] = await tx.execute<{ recent: boolean; person: boolean }>(sql`select ${rule.recent} as recent, ${rule.personAsked} as person`); + personAsked = decision?.person === true; + if (!personAsked && decision?.recent !== true) return { kind: "person_owned", reason: "retry_needed" }; + } + // Every wait is bounded in time: about 10 minutes from the first pending // check, or 15 while the exact-recovery owner shows progress (a new run // event, output or lease renewal since the last check). Then the fallback @@ -464,7 +480,8 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< // budget: it may still rescue unsaved files. Anything idle gets the normal bound. const savedWorkAdvancing = progressing && savedWork.some(recovery => recovery!.state === "running"); const waitBounded = savedWorkAdvancing ? waitedMs >= FRESH_CONTINUATION_SAVED_WORK_BOUND_MS - : waitedMs >= FRESH_CONTINUATION_PROGRESS_BOUND_MS || (waitedMs >= FRESH_CONTINUATION_PENDING_BOUND_MS && !progressing); + : waitedMs >= FRESH_CONTINUATION_PROGRESS_BOUND_MS || (waitedMs >= FRESH_CONTINUATION_PENDING_BOUND_MS && !progressing) + || (personAsked && !savedWork.some(recovery => EXACT_RECOVERY_STATES.includes(recovery!.state))); // Rechecks back off 1, 2, 3, 4 minutes, so the bound is reached on time. const step = Math.min(4, (typeof pendingMarker?.checks === "number" ? pendingMarker.checks : 0) + 1) * EXACT_RECOVERY_RECHECK_MS; const pending = (reason: string, retryAt: Date): FreshContinuationResult | null => waitBounded ? null @@ -477,6 +494,8 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.id} <> ${run.id}`, or(eq(heartbeatRuns.nativeIssueId, issue.id), eq(heartbeatRuns.issueId, issue.id), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`), + sql`not coalesce(${heartbeatRuns.status} = 'cancelled' and ${heartbeatRuns.startedAt} is null + and ${heartbeatRuns.errorCode} = 'execution_reconciliation_required', false)`, or(inArray(heartbeatRuns.status, LIVE_RUN_STATUSES), sql`(${heartbeatRuns.createdAt}, ${heartbeatRuns.id}) > (${run.createdAt.toISOString()}::timestamptz, ${run.id}::uuid)`))).limit(1); if (newer || typeof object(coordinator?.failureDetail).successorRunId === "string" @@ -576,6 +595,7 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< let savedInput: { id: string } | undefined; for (const wake of savedWakes) { const commentIds = queuedCommentIdsFromWakePayload(wake.payload); + if (refused && !commentIds.length && object(wake.payload).taskRetry) continue; if (!commentIds.length || (await undeliveredLegacyUserCommentIds(tx, companyId, issue.id, run.agentId, commentIds)).length) { savedInput = wake; break; @@ -595,12 +615,7 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< const fenced = await fenceSourceWorkspace(tx, run); // Never claim a stop that was not proven: record what isolates the successor. const isolation: FreshContinuationIsolation = fenced || remoteSource ? "remote_epoch" - : writer === "ended" && !waitBounded ? "writer_ended" : "fresh_isolated_workspace"; - // Only a run that carries the isolation in its context prepares the new - // workspace (create_new). Saved input carries no such marker: on a task with - // no project it would be refused at admission once the binding is cleared, - // and otherwise could resolve the old directory. So the recorded - // continuation runs first, binds the new workspace, and saved input follows. + : writer === "ended" && !waitBounded && !refused ? "writer_ended" : "fresh_isolated_workspace"; if (isolation === "fresh_isolated_workspace" && delivery === "saved_input") delivery = "recorded"; const unverified = await unverifiedActions(tx, run); const created = await createdObjects(tx, run); @@ -628,8 +643,17 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< // action the new run wants goes through its approval gate again. const settlement = { schema: FRESH_CONTINUATION_SCHEMA, actorType: "system", actorId: FRESH_CONTINUATION_ACTOR, decision: "continue_in_fresh_run", replay: "none", sourceRunId: run.id, failureCode: run.errorCode ?? null, - reason: "exact_resumption_unavailable", isolation, unverifiedActions: unverified, recordedAt: now.toISOString() }; - const ownerHold = owned.find(hold => issue.status === "blocked" && holdOwnsBlockedStatus(hold, issue, run)) ?? owned.at(-1); + reason: "exact_resumption_unavailable", isolation, unverifiedActions: unverified, recordedAt: now.toISOString(), + ...(owned.some(isRestoreHold) ? { heldFiles: "kept_not_applied" } : {}), + ...(refused ? { refusedAtDispatchRunId: refused.id, requestedBy: personAsked ? "person_retry" : "system" } : {}) }; + if (refused && delivery === "recorded") { + await tx.update(agentWakeupRequests).set({ status: "coalesced", finishedAt: now, updatedAt: now, + payload: sql`coalesce(${agentWakeupRequests.payload}, '{}'::jsonb) || ${JSON.stringify({ coalescedIntoFreshContinuationOf: run.id })}::jsonb` }) + .where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.requestedByActorType, "user"), isNull(agentWakeupRequests.runId), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`)); + } + const ownerHold =owned.find(hold => issue.status === "blocked" && holdOwnsBlockedStatus(hold, issue, run)) ?? owned.at(-1); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionReplacement); @@ -805,12 +829,33 @@ function liveDispositionHoldSql() { return and(inArray(issueRecoveryActions.status, ["active", "escalated"]), eq(issueRecoveryActions.kind, "active_run_watchdog"))!; } +const cancelledAtDispatch = (alias: "newer" | "held") => sql.raw(`coalesce(${alias}.status = 'cancelled' and ${alias}.started_at is null + and ${alias}.error_code = 'execution_reconciliation_required', false)`); +const runOfTask = (alias: "newer" | "held", issueId: SQL) => sql`(${sql.raw(alias)}.native_issue_id::text = ${issueId}::text + or ${sql.raw(alias)}.issue_id::text = ${issueId}::text or ${sql.raw(alias)}.context_snapshot->>'issueId' = ${issueId}::text)`; + +function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQL; sourceRunId: SQL }) { + const since = sql`${new Date(input.now.getTime() - BACKLOG_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz`; + const source = (condition: SQL) => sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${input.companyId} + and source.id::text = ${input.sourceRunId}::text and ${condition})`; + const refusedLater = sql`held.company_id = source.company_id and held.id <> source.id and ${runOfTask("held", input.issueId)} + and (held.created_at, held.id) > (source.created_at, source.id) and ${cancelledAtDispatch("held")}`; + const personAsked = source(sql`(exists (select 1 from ${agentWakeupRequests} retry where retry.company_id = source.company_id + and retry.status = 'deferred_issue_execution' and retry.reason = 'task_retry' and retry.requested_by_actor_type = 'user' + and retry.run_id is null and retry.payload->>'issueId' = ${input.issueId}::text and retry.requested_at > source.finished_at) + or exists (select 1 from ${heartbeatRuns} held where ${refusedLater} + and held.context_snapshot->>'wakeReason' = 'task_retry' and held.finished_at > ${since}))`); + const recent = source(sql`source.error_code in (${sql.join(BACKLOG_CONTINUATION_SOURCE_CODES.map(code => sql`${code}`), sql`, `)}) + and greatest(source.finished_at, (select max(held.finished_at) from ${heartbeatRuns} held where ${refusedLater})) > ${since}`); + return { recent, personAsked }; +} + /** The backlog pass's eligibility: the one predicate its listing and its deadline query share. */ function recentInterruptionSourceSql(now: Date) { - return sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} - and source.id::text = coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId') - and source.finished_at > ${new Date(now.getTime() - BACKLOG_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz - and source.error_code in (${sql.join(BACKLOG_CONTINUATION_SOURCE_CODES.map(code => sql`${code}`), sql`, `)}))`; + const { recent, personAsked } = interruptionEligibility({ now, companyId: sql`${issueRecoveryActions.companyId}`, + issueId: sql`${issueRecoveryActions.sourceIssueId}`, + sourceRunId: sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')` }); + return sql`(${recent} or ${personAsked})`; } /** Held records the backlog pass leaves for a person (older, or not an interruption). */ @@ -851,13 +896,13 @@ function heldExecutionWhere(options: { includeActive?: boolean } = {}) { sql`${issues.status} not in ('done', 'cancelled')`, sql`${issues.assigneeAgentId} is not null`, // Only the task's latest run, unfinished and still its assignee's: other - // holds are history, and are never touched here. sql`exists (select 1 from ${heartbeatRuns} source where source.company_id = ${issueRecoveryActions.companyId} and source.id::text = coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId') and source.status in ('failed', 'timed_out', 'interrupted', 'cancelled') and source.finished_at is not null and source.agent_id = ${issues.assigneeAgentId} and not exists (select 1 from ${heartbeatRuns} newer where newer.company_id = source.company_id and newer.id <> source.id and (newer.native_issue_id = ${issues.id} or newer.issue_id = ${issues.id} or newer.context_snapshot->>'issueId' = ${issues.id}::text) + and not ${cancelledAtDispatch("newer")} and (newer.status in ('queued', 'running', 'scheduled_retry') or (newer.created_at, newer.id) > (source.created_at, source.id))))`, )!; } diff --git a/server/src/services/fresh-continuation-retry-fold.ts b/server/src/services/fresh-continuation-retry-fold.ts new file mode 100644 index 00000000000..f09509b581e --- /dev/null +++ b/server/src/services/fresh-continuation-retry-fold.ts @@ -0,0 +1,31 @@ +import { and, eq, inArray, isNull, sql } from "drizzle-orm"; +import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db"; +import { queuedCommentIdsFromWakePayload } from "./issue-queued-comment-queue.js"; + +const LIVE_CONTINUATION_STATUSES = ["queued", "running"]; + +function retrySourceRunId(payload: unknown): string | null { + const taskRetry = (payload as { taskRetry?: { sourceRunId?: unknown } } | null)?.taskRetry; + return typeof taskRetry?.sourceRunId === "string" ? taskRetry.sourceRunId : null; +} + +export async function foldWaitingRetriesIntoContinuation(tx: Db, input: { + companyId: string; issueId: string; runId: string; sourceRunId: string | null; +}) { + if (!input.sourceRunId) return; + const [continuation] = await tx.select({ status: heartbeatRuns.status }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId))); + if (!continuation || !LIVE_CONTINUATION_STATUSES.includes(continuation.status)) return; + const waiting = await tx.select({ id: agentWakeupRequests.id, payload: agentWakeupRequests.payload }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.requestedByActorType, "user"), + isNull(agentWakeupRequests.runId), sql`${agentWakeupRequests.payload}->>'issueId' = ${input.issueId}`)).for("update"); + const retries = waiting.filter(wake => queuedCommentIdsFromWakePayload(wake.payload).length === 0 + && retrySourceRunId(wake.payload) === input.sourceRunId).map(wake => wake.id); + if (!retries.length) return; + const now = new Date(); + await tx.update(agentWakeupRequests).set({ status: "coalesced", runId: input.runId, finishedAt: now, updatedAt: now, + payload: sql`coalesce(${agentWakeupRequests.payload}, '{}'::jsonb) || ${JSON.stringify({ coalescedIntoFreshContinuationRun: input.runId })}::jsonb` }) + .where(and(eq(agentWakeupRequests.companyId, input.companyId), inArray(agentWakeupRequests.id, retries), + eq(agentWakeupRequests.status, "deferred_issue_execution"))); +} diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index a0a15712898..3e25cddcb02 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -4,6 +4,7 @@ import { verifyBlockerStallWake } from "../recovery/blocker-stall-wakeups.js"; import { CONTEXT_COMPACTION_WAKE_REASON, ContextCompactionError, isContextCompactionRun, readContextCompactionRequest } from "../context-compaction.js"; import { isProcessRunAdmissionClosed } from "./run-admission.js"; import { continueFreshInTransaction, hasPendingFreshContinuation, isFreshIsolatedContinuationWake, settleCoalescedFreshContinuationHolds } from "../fresh-continuation-fallback.js"; +import { foldWaitingRetriesIntoContinuation } from "../fresh-continuation-retry-fold.js"; import { refreshRunStartYieldIfStale } from "../run-start-handover.js"; import { deliveredQueuedCommentIds } from "../queued-comment-delivery.js"; import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, recheckQueuedRunInteractions, withInteractionDeliveryOrigin } from "../agent-interaction-delivery.js"; @@ -5082,6 +5083,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende eq(issueRecoveryActions.id, String(enrichedContextSnapshot.recoveryActionId)), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`)); } + if (readNonEmptyString(enrichedContextSnapshot.freshContinuationIsolation)) + await foldWaitingRetriesIntoContinuation(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, runId: newRun.id, + sourceRunId: readNonEmptyString(enrichedContextSnapshot.previousRunId) }); await tx .update(agentWakeupRequests) @@ -5365,6 +5369,13 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .returning() .then((rows) => rows[0]); + const continuedIssueId = readNonEmptyString(enrichedContextSnapshot.issueId); + if (continuedIssueId && readNonEmptyString(enrichedContextSnapshot.freshContinuationIsolation)) { + await tx.execute(sql`select id from issues where company_id = ${agent.companyId} and id = ${continuedIssueId} for update`); + await foldWaitingRetriesIntoContinuation(tx as unknown as Db, { companyId: agent.companyId, issueId: continuedIssueId, runId: newRun.id, + sourceRunId: readNonEmptyString(enrichedContextSnapshot.previousRunId) }); + } + await tx .update(agentWakeupRequests) .set({ diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index a8f411ff0ba..b786d4cf7f0 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -35,6 +35,8 @@ import { hasVerifiedWorkspaceRepair } from "./workspace-recovery-receipt.js"; import { recordLegacyShutdownBeforeDispatch, SERVER_SHUTDOWN_INTERRUPTED_CODE } from "./legacy-shutdown-before-dispatch.js"; type Run = typeof heartbeatRuns.$inferSelect; +export const UNRESTORED_FILES_NEXT_ACTION = "The task's saved files could not be put back after the run stopped. " + + "They are kept where they were and are not applied. Press Retry to continue from the last saved version in a new workspace."; export const LEGACY_RECOVERY_CAUSE = "legacy_execution_requires_reconciliation"; const unusedReconciliationContextKeys = [ @@ -244,7 +246,7 @@ export async function terminalizeLegacyExecution(input: { const values = { status: "resolved", outcome: "blocked", ownerType: "board", ownerAgentId: null, ownerUserId: null, returnOwnerAgentId: null, evidence, resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, - nextAction: "The original sandbox is retained for workspace repair. Verify its exact stop receipt, recover the missing files, and record workspaceRepairEvidence without changing the current task. Repair does not replay the stopped run. Explicitly remove the retained allocation after recovery.", + nextAction: UNRESTORED_FILES_NEXT_ACTION, }; if (existing) await tx.update(issueRecoveryActions).set(values).where(eq(issueRecoveryActions.id, existing.id)); else await tx.insert(issueRecoveryActions).values({ ...values, companyId: run.companyId, @@ -324,10 +326,9 @@ export async function terminalizeLegacyExecution(input: { attempt: executionFailureRetryCount(run) + 1, }, nextAction: hasRequiredWorkspaceRecovery(updated.resultJson) - ? "The original sandbox is retained for workspace repair. Verify that it has stopped, recover the missing files, record workspaceRepairEvidence, and explicitly remove the retained allocation when recovery is complete. A new agent turn cannot prove file recovery." - : hasWorkspaceRestoreFailure(updated.resultJson) - ? "Verify safe workspace staging or repair, then reconcile the stopped run before continuing. Saved work and approval decisions remain in force." - : "Inspect the stopped provider and recorded actions, then reconcile their outcomes before continuing. This adapter has not established a safe resume checkpoint.", + || hasWorkspaceRestoreFailure(updated.resultJson) ? UNRESTORED_FILES_NEXT_ACTION + : "The run stopped before it finished, and Paperclip cannot tell which of its last actions took effect. " + + "Press Retry to continue the task in a fresh run; the agent checks those actions before repeating any.", maxAttempts: 3, wakePolicy: null, supersedeOnIdentityChange: true, diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 7e786b071f6..e9c5e148fd1 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -175,6 +175,7 @@ import { buildExecutionReviewParticipantRecoveryNoticeSeed, buildExecutionReviewParticipantUnavailableNoticeSeed, buildStrandedRecoveryEscalationNotice, + buildNoRunningExecutionNoticeSeed, RETRY_FROM_SAVED_WORK_NEXT_ACTION, type StrandedRecoveryNoticeSeed, } from "./stranded-notice.js"; @@ -242,6 +243,8 @@ const RESOLVED_DEPENDENCY_WAKE_BACKSTOP_CANDIDATE_LIMIT = 500; // every heartbeat and would otherwise trigger a recovery issue after just two // productive heartbeats. Floor the override at 60s to keep the exemption from // being effectively disabled by misconfiguration. +const RECOVERY_TASK_NOT_NESTED = "This task was created to recover another task, so Paperclip does not create another recovery task for it."; + export const STRANDED_RECENT_PROGRESS_EXEMPTION_MS = Math.max( 60_000, Number(process.env.STRANDED_RECENT_PROGRESS_EXEMPTION_MS) || 30 * 60 * 1000, @@ -895,12 +898,11 @@ function isOperatorCancelledRun( ); } -function nonRetryableContinuationNotice(errorCode: string | null) { +function nonRetryableContinuationNotice() { return { body: - "Paperclip detected a non-retryable failure on this issue's continuation run " + - `(\`${errorCode}\`). Skipping automatic retries and moving it to \`blocked\` ` + - "so it is visible for intervention.", + "The agent's run stopped for a reason that retrying alone would not fix, so Paperclip did not retry it. " + + "Fix the reason shown in the details, then press Retry.", title: "Continuation failed", tone: "danger" as const, }; @@ -2508,14 +2510,12 @@ export function recoveryService( .then((rows) => rows[0] ?? null) : null; const sourceLine = sourceIssue - ? `- Original source issue: ${issueUiLink(sourceIssue, prefix)}` - : sourceIssueId - ? `- Original source issue: \`${sourceIssueId}\`` - : "- Original source issue: unknown"; + ? `- Original task: ${issueUiLink(sourceIssue, prefix)}` + : "- Original task: not found"; return [ "", - "- Nested recovery: suppressed because this issue is already a `stranded_issue_recovery` issue.", + `- ${RECOVERY_TASK_NOT_NESTED}`, sourceLine, "- Next action: press Retry on the original task to start it again from the saved work, then close this task.", ].join("\n"); @@ -2874,24 +2874,20 @@ export function recoveryService( input.prefix, ) : "none"; - const retryReason = - readNonEmptyString( - parseObject(input.latestRun?.contextSnapshot)?.retryReason, - ) ?? "none"; const failureSummary = summarizeRunFailureForIssueComment(input.latestRun); + const plain = (value: string) => value.replaceAll("_", " "); return [ - "Paperclip stopped automatic stranded-work recovery for this recovery issue.", + "Paperclip stopped trying to recover this task automatically.", "", - `- Recovery issue: ${issueUiLink({ identifier: input.issue.identifier, id: input.issue.id }, input.prefix)}`, - `- Previous status: \`${input.previousStatus}\``, + `- Recovery task: ${issueUiLink({ identifier: input.issue.identifier, id: input.issue.id }, input.prefix)}`, + `- Previous status: ${plain(input.previousStatus)}`, `- Latest run: ${runLink}`, - `- Latest run status: \`${input.latestRun?.status ?? "unknown"}\``, - `- Retry reason: \`${retryReason}\``, + `- Latest run status: ${plain(input.latestRun?.status ?? "unknown")}`, failureSummary ? `- Failure: ${failureSummary.trim()}` : "- Failure: none recorded", - "- Guard: recovery issues do not create nested `stranded_issue_recovery` issues.", + `- ${RECOVERY_TASK_NOT_NESTED}`, "", "Next action: press Retry to start this task again from the saved work.", ].join("\n"); @@ -5348,9 +5344,8 @@ export function recoveryService( previousStatus: issue.status as StrandedPreviousStatus, latestRun: latestPostResolutionRun, comment: - `Paperclip stopped requeueing accepted interaction \`${acceptedContinuationInteraction.id}\` after ` + - `${consecutive} consecutive continuation wakes were cancelled while waiting on review. ` + - "Moving the issue to `blocked` so the missing execution path is visible for intervention.", + `Paperclip stopped trying to continue this task after ${consecutive} attempts in a row were cancelled ` + + "while it waited for review. Press Retry to start it again.", }); if (updated) { result.escalated += 1; @@ -5633,14 +5628,7 @@ export function recoveryService( issue, previousStatus: "todo", latestRun, - notice: { - body: - "Paperclip automatically retried dispatch for this assigned `todo` issue after a lost wake/run, " + - "but it still has no live execution path. " + - "Moving it to `blocked` so it is visible for intervention.", - title: "No live execution path", - tone: "danger", - }, + notice: buildNoRunningExecutionNoticeSeed({ status: "todo" }), }); if (updated) { result.escalated += 1; @@ -5682,14 +5670,7 @@ export function recoveryService( issue, previousStatus: "todo", latestRun, - notice: { - body: - "Paperclip automatically retried dispatch for this assigned `todo` issue after a lost wake/run, " + - "but the bounded retry budget is spent and it still has no live execution path. " + - "Moving it to `blocked` so it is visible for intervention.", - title: "No live execution path", - tone: "danger", - }, + notice: buildNoRunningExecutionNoticeSeed({ status: "todo" }), }); if (updated) { result.escalated += 1; @@ -5816,9 +5797,7 @@ export function recoveryService( issue, previousStatus: "in_progress", latestRun: successfulRun, - comment: - "Paperclip automatically retried continuation for this assigned `in_progress` issue and the retry " + - "made progress, but it still has no live execution path. Moving it to `blocked` so it is visible for intervention.", + notice: buildNoRunningExecutionNoticeSeed({ status: "in_progress" }), }); if (updated) { result.escalated += 1; @@ -5895,7 +5874,7 @@ export function recoveryService( issue, previousStatus: "in_progress", latestRun, - notice: nonRetryableContinuationNotice(classification.errorCode), + notice: nonRetryableContinuationNotice(), }); if (updated) { result.escalated += 1; @@ -5915,20 +5894,11 @@ export function recoveryService( classification.errorCode, ); if (consecutive >= classification.maxAttempts) { - const attemptCopy = - consecutive <= 1 ? "" : ` (${consecutive}× attempts)`; const updated = await escalateStrandedAssignedIssue({ issue, previousStatus: "in_progress", latestRun, - notice: { - body: - "Paperclip automatically retried continuation for this assigned `in_progress` issue after its live " + - `execution disappeared, but it still has no live execution path${attemptCopy}. ` + - "Moving it to `blocked` so it is visible for intervention.", - title: "No live execution path", - tone: "danger", - }, + notice: buildNoRunningExecutionNoticeSeed({ status: "in_progress", attempts: consecutive }), }); if (updated) { result.escalated += 1; @@ -5985,18 +5955,9 @@ export function recoveryService( issue, previousStatus: "in_progress", latestRun, - notice: recoveryOutcome.retryExhausted ? { - body: - "Paperclip retried this issue's run after it ended without finishing, but the bounded retry budget " + - "is spent and it still has no live execution path. " + - "Moving it to `blocked` so it is visible for intervention.", - title: "No live execution path", - tone: "danger", - } : recoveryOutcome.continuationRefused ? CONTINUATION_REFUSED_NOTICE - // The scheduler declined the preparation retry for another - // reason (for example a durable owner or a permanent refusal): - // escalate it as a non-retryable failure, as before. - : nonRetryableContinuationNotice(latestRun?.errorCode ?? null), + notice: recoveryOutcome.retryExhausted ? buildNoRunningExecutionNoticeSeed({ status: "in_progress" }) + : recoveryOutcome.continuationRefused ? CONTINUATION_REFUSED_NOTICE + : nonRetryableContinuationNotice(), }); if (updated) { result.escalated += 1; diff --git a/server/src/services/recovery/stranded-notice.test.ts b/server/src/services/recovery/stranded-notice.test.ts index 30b540b37e5..6c6e932714d 100644 --- a/server/src/services/recovery/stranded-notice.test.ts +++ b/server/src/services/recovery/stranded-notice.test.ts @@ -7,6 +7,7 @@ import { buildExecutionReviewParticipantRecoveryNoticeSeed, buildExecutionReviewParticipantUnavailableNoticeSeed, buildImmediateExecutionPathRecoveryNoticeSeed, + buildNoRunningExecutionNoticeSeed, buildStrandedRecoveryEscalationNotice, buildWorkspaceValidationRecoveryNoticeSeed, } from "./stranded-notice.js"; @@ -16,9 +17,19 @@ function allRows(metadata: { sections: Array<{ rows: unknown[] }> }) { } describe("stranded recovery notice seeds", () => { + it("says what happened and the one thing to do, with no internal words (the UI recognizes this wording)", () => { + for (const seed of [buildImmediateExecutionPathRecoveryNoticeSeed({ status: "todo" }), + buildNoRunningExecutionNoticeSeed({ status: "in_progress", attempts: 3 })]) { + expect(seed.body).toMatch(/^Paperclip tried to (start|continue) this task again, but it still could not (start|continue)\b/); + expect(seed.body).toContain("Press Retry"); + expect(seed.body).not.toMatch(/execution path|intervention|`blocked`|`todo`|`in_progress`|issue|board operator|inspect|repair/i); + } + expect(buildNoRunningExecutionNoticeSeed({ status: "in_progress", attempts: 3 }).body).toContain("after 3 tries"); + }); + it.each([ - ["todo dispatch", buildImmediateExecutionPathRecoveryNoticeSeed({ status: "todo" }), "No live execution path"], - ["in_progress continuation", buildImmediateExecutionPathRecoveryNoticeSeed({ status: "in_progress" }), "No live execution path"], + ["todo dispatch", buildImmediateExecutionPathRecoveryNoticeSeed({ status: "todo" }), "Task could not continue"], + ["in_progress continuation", buildImmediateExecutionPathRecoveryNoticeSeed({ status: "in_progress" }), "Task could not continue"], ["workspace validation", buildWorkspaceValidationRecoveryNoticeSeed(), "Workspace validation failed"], ["configuration incomplete", buildConfigurationIncompleteRecoveryNoticeSeed(), "Configuration incomplete"], ["review participant recovery", buildExecutionReviewParticipantRecoveryNoticeSeed(), "Review recovery stalled"], @@ -117,9 +128,9 @@ describe("stranded recovery notice seeds", () => { }); it("distinguishes todo dispatch from in_progress continuation copy", () => { - expect(buildImmediateExecutionPathRecoveryNoticeSeed({ status: "todo" }).body).toContain("retried dispatch"); + expect(buildImmediateExecutionPathRecoveryNoticeSeed({ status: "todo" }).body).toContain("tried to start this task again"); expect(buildImmediateExecutionPathRecoveryNoticeSeed({ status: "in_progress" }).body).toContain( - "retried continuation", + "tried to continue this task again", ); }); }); diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index c0185da6783..65a321b34a9 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -87,19 +87,27 @@ const SETUP_FAILURE_NOTICES: Record = { }, }; +export const NO_RUNNING_EXECUTION_NOTICE_TITLE = "Task could not continue"; + +export function buildNoRunningExecutionNoticeSeed(input: { + status: "todo" | "in_progress"; + attempts?: number; +}): StrandedRecoveryNoticeSeed { + const tries = input.attempts && input.attempts > 1 ? ` after ${input.attempts} tries` : ""; + return { + body: input.status === "todo" + ? `Paperclip tried to start this task again, but it still could not start${tries}. Press Retry to start it again.` + : `Paperclip tried to continue this task again, but it still could not continue${tries}. ` + + "Press Retry to start it again from the saved work.", + title: NO_RUNNING_EXECUTION_NOTICE_TITLE, + tone: "danger", + }; +} + export function buildImmediateExecutionPathRecoveryNoticeSeed(input: { status: "todo" | "in_progress"; }): StrandedRecoveryNoticeSeed { - const retryDescription = input.status === "todo" - ? "Paperclip automatically retried dispatch for this assigned `todo` issue during terminal run recovery" - : "Paperclip automatically retried continuation for this assigned `in_progress` issue during terminal run recovery"; - return { - body: - `${retryDescription}, but it still has no live execution path. ` + - "Moving it to `blocked` so it is visible for intervention.", - title: "No live execution path", - tone: "danger", - }; + return buildNoRunningExecutionNoticeSeed(input); } export function buildWorkspaceValidationRecoveryNoticeSeed(): StrandedRecoveryNoticeSeed { diff --git a/server/src/services/system-recovery-state.test.ts b/server/src/services/system-recovery-state.test.ts index 0be11845f5b..4dae47be2e6 100644 --- a/server/src/services/system-recovery-state.test.ts +++ b/server/src/services/system-recovery-state.test.ts @@ -70,7 +70,8 @@ describe("durable system recovery budget", () => { expect(isDeterministicRecoveryRefusal(refused)).toBe(true); const text = exhaustedSystemRecoveryAction(refused); expect(text).toContain("Paperclip refused to recover the saved files"); - expect(text).toContain("Paperclip cannot prove which runs used the sandbox that holds the saved files (allocation_ownership_ambiguous)"); + expect(text).toContain("Paperclip cannot prove which runs used the sandbox that holds the saved files."); + expect(text).not.toContain("allocation_ownership_ambiguous"); expect(text).toContain("Retry runs the check again. The agent's result is saved"); // Recovery never waits on a person or a release, and never promises a // fallback the hold may not get. @@ -80,9 +81,9 @@ describe("durable system recovery budget", () => { const once = failSystemRecovery(reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!, new SystemRecoveryFailure("historical_source_missing", "use_alternative_strategy"), { now: 0 }); expect(isDeterministicRecoveryRefusal(once)).toBe(false); - expect(exhaustedSystemRecoveryAction(once)).toBe("Automatic recovery of saved work failed (last failure: historical_source_missing). " - + "Retry to try again; the agent will not repeat its work."); - expect(exhaustedSystemRecoveryAction({ ...once, attempts: 6 })).toContain("stopped after 6 attempts"); + expect(exhaustedSystemRecoveryAction(once)).toBe("Paperclip could not recover the agent's saved files. " + + "Press Retry to try again; the agent will not repeat its work."); + expect(exhaustedSystemRecoveryAction({ ...once, attempts: 6 })).toContain("after 6 tries"); }); it("promises an automatic retry for an unproven stop only while a re-arm remains", () => { @@ -96,13 +97,15 @@ describe("durable system recovery budget", () => { expect(exhaustedSystemRecoveryAction(stop)).toContain("Retry to try again"); }); - it("names the last failure and its cause in the board handoff", () => { + it("keeps the failure code and cause in the evidence, never in the text a person reads", () => { const state = reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!; const withCause = failSystemRecovery(state, new SystemRecoveryFailure("provider_unavailable", "retry_transient"), { now: 0, cause: { label: "sandbox_not_running", code: null, status: null } }); - expect(exhaustedSystemRecoveryAction(withCause)).toContain("(last failure: provider_unavailable, cause: sandbox_not_running)"); + expect(withCause.failure).toMatchObject({ code: "provider_unavailable", cause: { label: "sandbox_not_running" } }); + expect(exhaustedSystemRecoveryAction(withCause)).not.toMatch(/provider_unavailable|sandbox_not_running|last failure/); const withoutCause = failSystemRecovery(state, new SystemRecoveryFailure("archive_unsafe", "service_required"), { now: 0 }); - expect(exhaustedSystemRecoveryAction(withoutCause)).toContain("(archive_unsafe)"); + expect(exhaustedSystemRecoveryAction(withoutCause)).not.toContain("archive_unsafe"); + expect(exhaustedSystemRecoveryAction(withoutCause)).toContain("the saved file archive contains entries that are not safe to apply."); }); it("reconciles a recorded attempt after restart without spending another retry", () => { const running = reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!; diff --git a/server/src/services/system-recovery-state.ts b/server/src/services/system-recovery-state.ts index ffe8e22dfa7..fd250dfe189 100644 --- a/server/src/services/system-recovery-state.ts +++ b/server/src/services/system-recovery-state.ts @@ -193,7 +193,6 @@ export function exhaustedSystemRecoveryAction(state: SystemRecoveryState, eviden if (evidence !== undefined && isExhaustedStopProofWait(state) && !stopProofRearmsSpent(evidence)) return "Recovery of saved work is waiting to confirm that the previous run stopped. " + "Paperclip tries again automatically, and the agent will not repeat its work."; - const cause = state.failure?.cause ? `, cause: ${state.failure.cause.label}` : ""; if (state.failure?.cause?.label === "sandbox_disk_full") return "Saving the agent's files stopped because the sandbox disk is full. Paperclip deletes only inactive tool " + "caches it can prove safe, and that did not free enough space. Automatic recovery does not repeat a save that fails the same way. " @@ -202,14 +201,13 @@ export function exhaustedSystemRecoveryAction(state: SystemRecoveryState, eviden const code = state.failure!.code; // Never promise a release. A retained-file hold the fallback settles // (workspace-recovery-fallback) replaces this text with its own outcome. - return `Paperclip refused to recover the saved files: ${recoveryRefusalReasons[code] ?? "a safety check on the saved files did not pass"} ` - + `(${code}${cause}). Automatic recovery does not repeat this check, because the same saved state gives the same result. ` + return `Paperclip refused to recover the saved files: ${recoveryRefusalReasons[code] ?? "a safety check on the saved files did not pass"}. ` + + "Automatic recovery does not repeat this check, because the same saved state gives the same result. " + "Retry runs the check again. The agent's result is saved, and the agent will not repeat its work."; } - const reason = state.failure?.code ? ` (last failure: ${state.failure.code}${cause})` : ""; - const outcome = state.attempts > 1 ? `stopped after ${state.attempts} attempts` : "failed"; - return `Automatic recovery of saved work ${outcome}${reason}. ` - + "Retry to try again; the agent will not repeat its work."; + const tries = state.attempts > 1 ? ` after ${state.attempts} tries` : ""; + return `Paperclip could not recover the agent's saved files${tries}. ` + + "Press Retry to try again; the agent will not repeat its work."; } export function isDeterministicRecoveryRefusal(state: Pick | null | undefined) { diff --git a/server/src/services/workspace-recovery-warm-handoff.test.ts b/server/src/services/workspace-recovery-warm-handoff.test.ts index a97e26a569f..b2c13fcfa8d 100644 --- a/server/src/services/workspace-recovery-warm-handoff.test.ts +++ b/server/src/services/workspace-recovery-warm-handoff.test.ts @@ -546,7 +546,8 @@ describe("automatic retained workspace recovery", () => { const exhausted = await action(f.actionId); expect(readSystemRecovery(exhausted.evidence)?.state).toBe("exhausted"); expect(exhausted).toMatchObject({ status: "active", ownerType: "board" }); - expect(exhausted.nextAction).toMatch(/stopped after 6 attempts/); + expect(exhausted.nextAction).toMatch(/after 6 tries/); + expect(exhausted.nextAction).not.toContain("export_interrupted"); const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); expect(blocker?.workspaceRecovery).toMatchObject({ status: "blocked", canRetry: true }); expect(blocker?.nextAction).toMatch(/Retry to try again/); diff --git a/ui/src/components/IssueRecoveryActionCard.test.tsx b/ui/src/components/IssueRecoveryActionCard.test.tsx index 886a1491a93..b37514f1113 100644 --- a/ui/src/components/IssueRecoveryActionCard.test.tsx +++ b/ui/src/components/IssueRecoveryActionCard.test.tsx @@ -1131,7 +1131,8 @@ describe("IssueRecoveryActionCard owner-sticky retry lineage", () => { expect(section?.getAttribute("data-recovery-state")).toBe("needed"); expect(section?.getAttribute("data-recovery-lane")).toBe("board"); expect(node.textContent).toContain("Automatic recovery is exhausted"); - expect(node.textContent).toContain("Board decision required"); + expect(node.textContent).toContain("Waiting for you"); + expect(node.textContent).not.toContain("Board decision required"); const recoveryOwner = node.querySelector("[data-testid='recovery-recovery-owner']"); expect(recoveryOwner?.textContent).toContain("Board"); expect(recoveryOwner?.textContent).toContain("decides the next step only"); diff --git a/ui/src/components/IssueRecoveryActionCard.tsx b/ui/src/components/IssueRecoveryActionCard.tsx index 57dba70849f..ecbec1c8b72 100644 --- a/ui/src/components/IssueRecoveryActionCard.tsx +++ b/ui/src/components/IssueRecoveryActionCard.tsx @@ -771,10 +771,10 @@ function readWakePolicySummary(action: IssueRecoveryAction): string | null { if (type === "bounded_owner_disposition_repair") { return "Paperclip is retrying the original owner"; } - if (type === "bounded_recovery_owner") return "A recovery owner is repairing the next step"; - if (type === "board_escalation") return "Board decision required"; + if (type === "bounded_recovery_owner") return "An agent is choosing the next step"; + if (type === "board_escalation") return "Waiting for you"; if (type === "manual") return "Manual follow-up needed"; - if (type === "manual_repair_required") return "Repair needed before retry"; + if (type === "manual_repair_required") return "Needs a fix before Retry"; if (type === "monitor") { const interval = readEvidenceString(policy.intervalLabel); return interval ? `Check scheduled · ${interval}` : "Check scheduled"; diff --git a/ui/src/components/task-chat/adjacent-errors.test.ts b/ui/src/components/task-chat/adjacent-errors.test.ts index d1f12ec3d6c..beada087e71 100644 --- a/ui/src/components/task-chat/adjacent-errors.test.ts +++ b/ui/src/components/task-chat/adjacent-errors.test.ts @@ -47,6 +47,23 @@ describe("compactAdjacentErrors", () => { expect(compactAdjacentErrors([failed, legacy]).items).toEqual([legacy]); }); + it.each([ + "Paperclip tried to start this task again, but it still could not start. Press Retry to start it again.", + "Paperclip tried to continue this task again, but it still could not continue after 3 tries. Press Retry to start it again from the saved work.", + ])("recognizes the current recovery notice: %s", (text) => { + const titled = { ...recovery, text, presentation: { ...recovery.presentation!, title: "Task could not continue" } }; + expect(compactAdjacentErrors([failed, titled]).items).toEqual([titled]); + const unstructured = { ...recovery, text, presentation: null }; + expect(compactAdjacentErrors([failed, unstructured]).items).toEqual([unstructured]); + const retitled: TaskChatMessageItem = { ...recovery, text, + presentation: { ...recovery.presentation!, title: "Automatic recovery blocked" }, + metadata: { version: 1, sections: [{ title: "Recovery", rows: [ + { type: "key_value", label: "Recovery action", value: "recovery-action-1" }] }] } }; + expect(compactAdjacentErrors([failed, retitled]).items).toEqual([retitled]); + const quoted = { ...recovery, presentation: null, text: `Child task report: ${text}` }; + expect(compactAdjacentErrors([failed, quoted]).items).toEqual([failed, quoted]); + }); + it.each(["Automatic recovery blocked", "Error: usage limit reached"])( "recognizes a saved stranded-recovery notice with the producer title %s", (title) => { // buildStrandedRecoveryEscalationNotice saves this metadata even when diff --git a/ui/src/components/task-chat/adjacent-errors.ts b/ui/src/components/task-chat/adjacent-errors.ts index 3cf520086e1..184b661c72d 100644 --- a/ui/src/components/task-chat/adjacent-errors.ts +++ b/ui/src/components/task-chat/adjacent-errors.ts @@ -1,10 +1,15 @@ import type { TaskChatItem, TaskChatMarkerItem, TaskChatMessageItem } from "./task-chat-model"; +const RECOVERY_STATEMENT = /^Paperclip tried to (?:start|continue) this task again, but it still could not (?:start|continue)\b/i; +const LEGACY_RECOVERY_STATEMENT = /^Paperclip (?:automatically )?retried (?:dispatch|continuation|this issue's run)\b[^\r\n]*\bit still has no live execution path\b/i; +const RECOVERY_TITLES = new Set(["task could not continue", "no live execution path"]); + export function isNoLiveExecutionPathNotice(item: TaskChatMessageItem): boolean { if (item.author !== "system") return false; - const recoveryStatement = /^Paperclip (?:automatically )?retried (?:dispatch|continuation|this issue's run)\b[^\r\n]*\bit still has no live execution path\b/i.test(item.text.trim()); + const text = item.text.trim(); + const recoveryStatement = RECOVERY_STATEMENT.test(text) || LEGACY_RECOVERY_STATEMENT.test(text); if (item.presentation) { - if (item.presentation.title?.trim().toLowerCase() === "no live execution path") return true; + if (RECOVERY_TITLES.has(item.presentation.title?.trim().toLowerCase() ?? "")) return true; // The stranded-recovery producer can replace the family title with // "Automatic recovery blocked" or a classified source-run failure. // Require both its recovery statement and saved action metadata so an diff --git a/ui/src/lib/system-notice-humanizer.test.ts b/ui/src/lib/system-notice-humanizer.test.ts index 88d8787dcef..2f2ae6111b6 100644 --- a/ui/src/lib/system-notice-humanizer.test.ts +++ b/ui/src/lib/system-notice-humanizer.test.ts @@ -74,7 +74,15 @@ describe("humanizeSystemNotice", () => { "Paperclip automatically retried dispatch, but it still has no live execution path. " + "Moving it to `blocked` so it is visible for intervention.", }); - expect(result.title).toBe("Task paused — waiting on a recovery owner"); + expect(result.title).toBe("Task could not continue"); + }); + + it("humanizes the current recovery notice saved without a presentation title", () => { + const result = humanizeSystemNotice({ + body: "Paperclip tried to continue this task again, but it still could not continue. " + + "Press Retry to start it again from the saved work.", + }); + expect(result).toEqual({ title: "Task could not continue", tone: "warning" }); }); it("falls back to System update plus a truncated first sentence", () => { diff --git a/ui/src/lib/system-notice-humanizer.ts b/ui/src/lib/system-notice-humanizer.ts index 6b2177eada8..c4ddcb7adcf 100644 --- a/ui/src/lib/system-notice-humanizer.ts +++ b/ui/src/lib/system-notice-humanizer.ts @@ -74,12 +74,13 @@ export function humanizeSystemNotice(input: { tone: presentationTone ?? "warning", }; } - if (body.includes("no live execution path")) { + if (body.includes("no live execution path") + || /^Paperclip tried to (?:start|continue) this task again\b/.test(body.trim())) { const owner = recoveryOwnerName(body); return { title: owner ? `Task paused — waiting on ${owner}` - : "Task paused — waiting on a recovery owner", + : "Task could not continue", tone: presentationTone ?? "warning", }; }