From ce4431410b74dc285d6a0b8c2dafe3c2ce5ddf0f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 13 Nov 2026 15:37:01 -0600 Subject: [PATCH] fix(recovery): continue a task whose system retry was refused at claim behind a saved-work hold A follow-up run that failed before provider work left a saved-work hold. The dispatch gate cancelled the system's own retry, and the fallback then left the task for a Retry that was not offered. A system recovery run refused at dispatch now counts as the system continuing the task. Attempts that never started do not spend the hourly budget. The sweep continues a blocked task behind a refused run. Notices never ask for a Retry the task does not offer. Co-Authored-By: Claude Opus 5.5 --- ...ailed-continuation-continues-again.test.ts | 32 +++++ .../fresh-continuation-fallback.test.ts | 123 ++++++++++++++++++ .../services/fresh-continuation-fallback.ts | 7 +- .../services/recovery/assigned-work-sweep.ts | 33 ++++- .../services/recovery/continuation-ledger.ts | 8 +- server/src/services/recovery/service.ts | 10 ++ .../services/recovery/stranded-notice.test.ts | 13 ++ .../src/services/recovery/stranded-notice.ts | 15 ++- .../recovery/task-automatic-attempts.ts | 5 +- 9 files changed, 235 insertions(+), 11 deletions(-) diff --git a/server/src/__tests__/failed-continuation-continues-again.test.ts b/server/src/__tests__/failed-continuation-continues-again.test.ts index ba7caebdcd2..0c62829be1e 100644 --- a/server/src/__tests__/failed-continuation-continues-again.test.ts +++ b/server/src/__tests__/failed-continuation-continues-again.test.ts @@ -414,4 +414,36 @@ describePostgres("a continuation that fails before doing any work is continued a expect(await sweep.checkIssue({ companyId: f.companyId, issueId: task.id })).toMatchObject({ outcome: "repaired" }); expect(await continuationsOf(stopped.id)).toHaveLength(1); }, 60_000); + + it("the sweep continues a blocked task whose system retry was refused at claim behind a recovered saved-work hold", async () => { + resetBacklogContinuationPacingForTests(db); + const f = await seed(); + const blockedAt = new Date(Date.now() - 20 * MINUTE); + const task = await f.task({ status: "blocked", blockedTransitionAt: blockedAt, statusVersion: 3 }); + const source = await f.run(task.id, { errorCode: "adapter_failed", error: "native_instruction_materialization_handoff_unavailable", + finishedAt: new Date(blockedAt.getTime() - 500), resultJson: { workspaceRestoreFailure: "restore_failed" }, + contextSnapshot: { wakeReason: "issue_children_completed" } }); + const [hold] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: task.id, kind: "active_run_watchdog", + status: "active", ownerType: "board", returnOwnerAgentId: f.agentId, cause: "legacy_execution_requires_reconciliation", + fingerprint: `legacy-execution:${source.id}`, nextAction: "Held.", createdAt: blockedAt, + evidence: { runId: source.id, originalFailureCode: "adapter_failed", workspaceRestoreFailure: "restore_failed", + systemRecovery: { ...createSystemRecovery(source.id), state: "complete" } } }).returning(); + const refused = await f.run(task.id, { status: "cancelled", errorCode: "execution_reconciliation_required", startedAt: null, + retryOfRunId: source.id, finishedAt: new Date(blockedAt.getTime() + 1_000), createdAt: new Date(blockedAt.getTime() - 100), resultJson: {}, + contextSnapshot: { wakeReason: "issue_continuation_needed", retryReason: "issue_continuation_needed" } }); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id)).toContain(task.id); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id })) + .toMatchObject({ outcome: "repaired", verdict: { kind: "violation" } }); + const [record] = await continuationsOf(source.id); + expect(record?.id).toBe(hold!.id); + expect(record?.evidence).toMatchObject({ continuationDelivery: "pending", freshContinuation: { sourceRunId: source.id, trigger: "assigned_work" }, + freshContinuationSettlement: { refusedAtDispatchRunId: refused.id, requestedBy: "system" } }); + expect((await taskRow(task.id)).status).toBe("todo"); + expect(queue.calls).toHaveLength(0); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id })) + .toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent" } }); + }, 60_000); }); diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index a0351244cf7..39763425afa 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -29,6 +29,8 @@ import { createPostgresRunDispatchAdapter } from "../modules/run-dispatch/adapte import { createSystemRecovery } from "./system-recovery-state.js"; import { requestTaskRetry } from "./task-retry.js"; import { terminalizeLegacyExecution } from "./legacy-execution-recovery.js"; +import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js"; +import { taskAutomaticAttempts } from "./recovery/task-automatic-attempts.js"; const support = await getEmbeddedPostgresTestSupport(); const suite = support.supported ? describe : describe.skip; @@ -1323,4 +1325,125 @@ suite("fresh-continuation fallback", () => { const [next] = await successors(f.runId); expect(await readFreshWorkspaceReadiness(db, next!)).toMatchObject({ mode: "fresh_isolated_workspace" }); }, 60_000); + + async function followUpFailedBeforeProviderWork(options: { restoreFailed?: boolean } = {}) { + const issueId = randomUUID(), runId = randomUUID(); + const endedAt = new Date(Date.now() - 60_000); + await db.insert(issues).values({ id: issueId, companyId, title: "Summarize the subtasks", status: "in_progress", assigneeAgentId: agentId }); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issueId, mode: "isolated", + strategyType: "git_worktree", name: "Task workspace", cwd: "/work/remote-task" }).returning(); + await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issueId)); + const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "automation", issueId, + runtimeMode: "legacy", status: "running", startedAt: new Date(endedAt.getTime() - 10_000), + createdAt: new Date(endedAt.getTime() - 60_000), responsibleUserId: userId, + contextSnapshot: { issueId, executionWorkspaceId: workspace!.id, wakeReason: "issue_children_completed" } }).returning(); + const [environment] = await db.insert(environments).values({ name: `sandbox-${runId}`, driver: "sandbox" }).returning(); + await db.insert(environmentLeases).values({ companyId, issueId, executionWorkspaceId: workspace!.id, + heartbeatRunId: runId, environmentId: environment!.id, status: "active", leasePolicy: "reuse_by_environment", provider: "daytona", + providerLeaseId: `sandbox-${runId}`, metadata: { remoteCwd: "/work/remote-task", driver: "sandbox", + workspaceRealization: { authoritativeRoot: "/work/remote-task" } } }); + await terminalizeLegacyExecution({ db, run: run!, status: "failed", patch: { errorCode: "adapter_failed", + error: "native_instruction_materialization_handoff_unavailable", finishedAt: endedAt, + resultJson: options.restoreFailed === false ? { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } + : { workspaceRestoreFailure: "restore_failed" } } }); + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + return { issueId, runId, actionId: hold?.id ?? null }; + } + + async function savedWorkRecoveryState(actionId: string, runId: string, state: "running" | "complete") { + await db.update(issueRecoveryActions).set({ updatedAt: new Date(), + evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ systemRecovery: { ...createSystemRecovery(runId), state } })}::jsonb` }) + .where(eq(issueRecoveryActions.id, actionId)); + } + + async function systemRetryAtClaim(f: { issueId: string; runId: string }) { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, + runtimeMode: "legacy", status: "queued", retryOfRunId: f.runId, + contextSnapshot: { issueId: f.issueId, wakeReason: "issue_continuation_needed", retryReason: "issue_continuation_needed", + retryOfRunId: f.runId } }).returning(); + const outcome = await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId, runId: run!.id, + expectedStatus: "queued", now: new Date() }); + return { run: run!, outcome }; + } + + async function deliveredContinuationIsClaimable(f: { issueId: string; runId: string; actionId: string | null }, refusedRunId: string) { + await heartbeatService(db).reconcileStrandedAssignedIssues({ companyId, issueId: f.issueId }); + expect((await issue(f.issueId)).status).not.toBe("blocked"); + expect(await getExecutionBlocker(db, companyId, f.issueId)).toBeNull(); + await occupySlot(); + await deliver(); + const next = (await successors(f.runId)).filter(run => run.id !== refusedRunId); + expect(next).toHaveLength(1); + expect(next[0]!.status).toBe("queued"); + expect(await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId, runId: next[0]!.id, + expectedStatus: "queued", now: new Date() })).toEqual({ outcome: "not_stale" }); + return next[0]!; + } + + it("continues a follow-up that failed before provider work with unrestored files, when the saved work was recovered before the system retry was refused at claim", async () => { + const f = await followUpFailedBeforeProviderWork(); + expect(f.actionId).not.toBeNull(); + await savedWorkRecoveryState(f.actionId!, f.runId, "complete"); + const refused = await systemRetryAtClaim(f); + expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" }); + expect((await holdRow(f.actionId!)).evidence).toMatchObject({ + freshContinuation: { sourceRunId: f.runId }, continuationDelivery: "pending", + freshContinuationSettlement: { refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } }); + await deliveredContinuationIsClaimable(f, refused.run.id); + }, 60_000); + + it("continues a follow-up that failed before provider work with unrestored files, when the saved work was recovered after the system retry was refused at claim", async () => { + const f = await followUpFailedBeforeProviderWork(); + await savedWorkRecoveryState(f.actionId!, f.runId, "running"); + const refused = await systemRetryAtClaim(f); + expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" }); + expect((await holdRow(f.actionId!)).evidence.freshContinuationCheck) + .toMatchObject({ kind: "exact_recovery_pending", reason: "saved_work_recovery_running" }); + await savedWorkRecoveryState(f.actionId!, f.runId, "complete"); + await makeDue(f.actionId!); + expect(await continueHeldExecutions(db)).toMatchObject({ continued: 1 }); + expect((await holdRow(f.actionId!)).evidence).toMatchObject({ freshContinuation: { sourceRunId: f.runId }, + freshContinuationSettlement: { refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } }); + await deliveredContinuationIsClaimable(f, refused.run.id); + }, 60_000); + + it("continues a follow-up that failed before provider work with its files in place, after its system retry was refused at claim", async () => { + const f = await followUpFailedBeforeProviderWork({ restoreFailed: false }); + const refused = await systemRetryAtClaim(f); + expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" }); + expect((await holdRow(f.actionId!)).evidence).toMatchObject({ freshContinuation: { sourceRunId: f.runId }, + freshContinuationSettlement: { refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } }); + await deliveredContinuationIsClaimable(f, refused.run.id); + }, 60_000); + + it("counts only automatic retries that started, or still may, against the task's hourly budget", async () => { + const f = await followUpFailedBeforeProviderWork(); + for (let index = 0; index < FRESH_CONTINUATIONS_PER_HOUR; index++) { + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, runtimeMode: "legacy", + status: "cancelled", errorCode: "execution_reconciliation_required", finishedAt: new Date(), retryOfRunId: f.runId, + scheduledRetryReason: "transient_failure", contextSnapshot: { issueId: f.issueId, wakeReason: "transient_failure_retry" } }); + } + expect(await sharedContinuationAttempts(db, companyId, f.issueId, new Date())).toHaveLength(0); + expect((await taskAutomaticAttempts(db, companyId, f.issueId, new Date())).started).toHaveLength(0); + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, runtimeMode: "legacy", + status: "failed", errorCode: "adapter_failed", startedAt: new Date(), finishedAt: new Date(), retryOfRunId: f.runId, + scheduledRetryReason: "transient_failure", contextSnapshot: { issueId: f.issueId, wakeReason: "transient_failure_retry" } }); + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, runtimeMode: "legacy", + status: "scheduled_retry", retryOfRunId: f.runId, + scheduledRetryReason: "transient_failure", contextSnapshot: { issueId: f.issueId, wakeReason: "transient_failure_retry" } }); + expect(await sharedContinuationAttempts(db, companyId, f.issueId, new Date())).toHaveLength(2); + expect((await taskAutomaticAttempts(db, companyId, f.issueId, new Date())).started).toHaveLength(2); + }, 60_000); + + it("still leaves a person's failed Retry of a follow-up for that person after its successor is refused at claim", async () => { + const f = await followUpFailedBeforeProviderWork(); + await savedWorkRecoveryState(f.actionId!, f.runId, "complete"); + await db.update(heartbeatRuns).set({ contextSnapshot: sql`${heartbeatRuns.contextSnapshot} || '{"wakeReason":"task_retry"}'::jsonb` }) + .where(eq(heartbeatRuns.id, f.runId)); + const refused = await systemRetryAtClaim(f); + expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" }); + expect((await holdRow(f.actionId!)).evidence.freshContinuation).toBeUndefined(); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "held_recovery" })) + .toEqual({ kind: "person_owned", reason: "retry_needed" }); + }, 60_000); }); diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 39cb5d8d485..f8ad3bb99d0 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -26,7 +26,7 @@ import { notifyRoutineExecutionStatus } from "./routine-execution-status-notific import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { readRunCancellation } from "./run-cancellation.js"; import { readSystemRecovery } from "./system-recovery-state.js"; -import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuesAutomaticallySql, isPersonOwnedFailureCode } from "./continuation-cause.js"; +import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, isPersonOwnedFailureCode, recoveryRunSql } from "./continuation-cause.js"; import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js"; export { AUTOMATIC_CONTINUATION_WAKE_REASONS, PERSON_OWNED_FAILURE_CODES, isPersonOwnedFailureCode, isRecoveryRun } from "./continuation-cause.js"; @@ -813,7 +813,10 @@ function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQ and retry.run_id is null and retry.payload->>'issueId' = ${input.issueId}::text and retry.requested_at > source.finished_at) or exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and held.context_snapshot->>'wakeReason' = 'task_retry' and held.finished_at > ${since}))`); - const recent = source(sql`${continuesAutomaticallySql("source")} + const systemRetriedSource = sql`(${continuationCauseIsSystemOwnedSql("source")} + and coalesce(source.context_snapshot->>'wakeReason', '') <> 'task_retry' and not coalesce(source.context_snapshot ? 'explicitUserContinuation', false) + and exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and ${recoveryRunSql("held")}))`; + const recent = source(sql`(${continuesAutomaticallySql("source")} or ${systemRetriedSource}) and greatest(source.finished_at, (select max(held.finished_at) from ${heartbeatRuns} held where ${refusedLater})) > ${since}`); return { recent, personAsked }; } diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 9128d888876..0a6a58bd133 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -466,6 +466,13 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { * Chat conversations and blocks older than the stale window are left out: * the first wait on a person's message, the second is only reported. */ + function latestRunRefusedAtDispatch(at: Date) { + return sql`coalesce((select lr.status = 'cancelled' and lr.started_at is null and lr.error_code = 'execution_reconciliation_required' + and lr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz + from heartbeat_runs lr where lr.company_id = ${issues.companyId} and (lr.context_snapshot->>'issueId') = ${issues.id}::text + order by lr.created_at desc limit 1), false)`; + } + async function listSuspects(scope: { companyId?: string; issueId?: string; createdAfter?: Date | null }, cursor: string | null, limit: number) { const at = now(); const liveRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} @@ -518,7 +525,9 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { -- except a recent board escalation, which the predicate judges by its failed run's cause and not (ra.kind = ${STRANDED_KIND} and ra.owner_type = 'board' and exists (select 1 from heartbeat_runs sr where sr.company_id = ra.company_id and sr.id::text = ra.evidence->>'latestRunId' - and sr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz)))`, + and sr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz)) + and not (${issues.status} = 'blocked' and ra.cause not in ('configuration_incomplete', 'ownership_owner_unprovable') + and ${latestRunRefusedAtDispatch(at)}))`, // An assignee that cannot take work (paused, being set up, held) is // its own wait; the predicate names it when anything else asks. sql`exists (select 1 from agents a where a.id = ${issues.assigneeAgentId} and a.company_id = ${issues.companyId} @@ -563,6 +572,11 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { // hold's owner (the fresh-continuation fallback, same per-task cap). The // sweep reports it and repairs only blocks that have no hold at all, so if (issue.status === "blocked" && facts.activeRecoveryAction && !facts.activeRecoveryAction.continuedBySweep) { + const refused = facts.latestRun?.status === "cancelled" && facts.latestRun.errorCode === "execution_reconciliation_required" + ? await continueFromSavedWork(issue, facts, at) : null; + if (refused?.kind === "continued" || refused?.kind === "already_continued") return { verdict, outcome: "repaired", retryAt: null }; + if (refused?.kind === "backoff" || refused?.kind === "exact_recovery_pending") + return { verdict, outcome: refused.kind === "backoff" ? "deferred_cap" : "waiting_on_recovery", retryAt: refused.retryAt }; return { verdict, outcome: "left_to_hold_owner", retryAt: null }; } // A stalled native run is ended by native recovery, not here: the sweep @@ -624,8 +638,21 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { async function continueFromSavedWork(issue: IssueRow, facts: AssignedWorkFacts, at: Date): Promise { const source = facts.latestRun; - if (!source?.finishedAt || !["failed", "timed_out", "interrupted", "cancelled"].includes(source.status) - || (source.status === "cancelled" && source.errorCode === "execution_reconciliation_required")) return null; + if (!source?.finishedAt || !["failed", "timed_out", "interrupted", "cancelled"].includes(source.status)) return null; + if (source.status === "cancelled" && source.errorCode === "execution_reconciliation_required") { + const [refusedFor] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, issue.companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`, + inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]), sql`${heartbeatRuns.finishedAt} is not null`, + sql`not (${heartbeatRuns.status} = 'cancelled' and ${heartbeatRuns.startedAt} is null + and coalesce(${heartbeatRuns.errorCode}, '') = 'execution_reconciliation_required')`)) + .orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); + if (!refusedFor) return null; + return ensureFreshContinuation(db, { companyId: issue.companyId, runId: refusedFor.id, trigger: "assigned_work", now: at }) + .catch(error => { + logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id }, "assigned-work continuation behind a refused run deferred"); + return null; + }); + } const [run] = await db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode, error: heartbeatRuns.error, retryOfRunId: heartbeatRuns.retryOfRunId, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, contextSnapshot: heartbeatRuns.contextSnapshot, runnerProfileJson: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns) diff --git a/server/src/services/recovery/continuation-ledger.ts b/server/src/services/recovery/continuation-ledger.ts index 995f1eed742..5b7ecb5a861 100644 --- a/server/src/services/recovery/continuation-ledger.ts +++ b/server/src/services/recovery/continuation-ledger.ts @@ -6,6 +6,11 @@ export const SCHEDULER_TRANSIENT_RETRY_REASON = "transient_failure"; export const AUTOMATIC_RETRY_REASONS = [SCHEDULER_TRANSIENT_RETRY_REASON, "server_restart", "interrupted_preparation"]; export const SHARED_CONTINUATION_WINDOW_MS = 60 * 60_000; +export function startedOrPendingAutomaticRetry() { + return sql`not (${heartbeatRuns.status} = 'cancelled' and ${heartbeatRuns.startedAt} is null + and coalesce(${heartbeatRuns.errorCode}, '') = 'execution_reconciliation_required')`; +} + export async function freshContinuationTimes(db: Db, companyId: string, issueId: string, since: Date) { const rows = await db.select({ recordedAt: sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}'`, trigger: sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,trigger}'` }).from(issueRecoveryActions).where(and( @@ -25,7 +30,8 @@ export async function sharedContinuationAttempts(db: Db, companyId: string, issu sql`coalesce(${activityLog.details}->>'continuedFromSavedWork', '') <> 'true'`)).limit(64), db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since))).limit(64), + inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since), + startedOrPendingAutomaticRetry())).limit(64), freshContinuationTimes(db, companyId, issueId, since), ]); return [...plainContinuations.map(row => row.createdAt), ...retries.map(row => row.createdAt), ...fresh.map(record => record.at)] diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 7bc2608bb8f..5c4cb40d923 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -176,6 +176,7 @@ import { buildExecutionReviewParticipantRecoveryNoticeSeed, buildExecutionReviewParticipantUnavailableNoticeSeed, buildStrandedRecoveryEscalationNotice, + withoutRetryInstruction, buildNoRunningExecutionNoticeSeed, RETRY_FROM_SAVED_WORK_NEXT_ACTION, type StrandedRecoveryNoticeSeed, @@ -185,6 +186,7 @@ import { isStrandedIssueRecoveryOriginKind, } from "./origins.js"; import { withRecoveryContext } from "./status-only-context.js"; +import { getExecutionBlocker } from "../execution-blocker.js"; import { currentRunFailureText } from "../native-runtime/native-exact-resume-failure.js"; import { isAutomaticRecoverySuppressedByPauseHold } from "./pause-hold-guard.js"; import { rehomeStrandedLegacyWorkspaceTasks } from "./legacy-workspace-rehome.js"; @@ -4250,6 +4252,13 @@ export function recoveryService( recoveryCause, successfulRunHandoffEvidence: input.successfulRunHandoffEvidence, }, tx); + const executionBlocker = await getExecutionBlocker(tx as unknown as Db, input.issue.companyId, input.issue.id); + const retryOffered = !(executionBlocker && executionBlocker.canRetry === false); + if (!retryOffered && /\bretry\b/i.test(recoveryAction.nextAction ?? "")) { + recoveryAction.nextAction = withoutRetryInstruction(recoveryAction.nextAction!); + await tx.update(issueRecoveryActions).set({ nextAction: recoveryAction.nextAction }) + .where(and(eq(issueRecoveryActions.companyId, input.issue.companyId), eq(issueRecoveryActions.id, recoveryAction.id))); + } const isProviderQuotaWait = recoveryCause === "provider_quota" && !recoveryAction.ownerAgentId && @@ -4333,6 +4342,7 @@ export function recoveryService( fallbackBody: input.comment, recoveryCause, recoveryActionId: recoveryAction.id, + retryOffered, continuesAutomatically: Boolean(input.latestRun) && recoveryAction.kind === STRANDED_KIND && recoveryAction.ownerType === "board" && input.latestRun?.agentId === input.issue.assigneeAgentId && strandedRunCauseIsTemporary({ diff --git a/server/src/services/recovery/stranded-notice.test.ts b/server/src/services/recovery/stranded-notice.test.ts index 6c6e932714d..9ed9e3d9eb6 100644 --- a/server/src/services/recovery/stranded-notice.test.ts +++ b/server/src/services/recovery/stranded-notice.test.ts @@ -401,3 +401,16 @@ it("tells the operator to switch a preconfigured Runner home to an isolated one" expect(notice.nextAction).toContain("isolated Runner home"); expect(notice.nextAction).not.toContain("Choose an AI connection"); }); + +it("never tells a person to press Retry when the task offers no Retry", () => { + const sourceRun = { id: "run-1", status: "cancelled", errorCode: "execution_reconciliation_required" }; + for (const seed of [buildNoRunningExecutionNoticeSeed({ status: "in_progress" }), null]) { + const offered = buildStrandedRecoveryEscalationNotice({ seed, recoveryActionId: "action-1", recoveryOwner: null, sourceRun }); + expect(`${offered.body} ${JSON.stringify(allRows(offered.metadata))}`).toMatch(/Press Retry/); + const withheld = buildStrandedRecoveryEscalationNotice({ seed, recoveryActionId: "action-1", recoveryOwner: null, sourceRun, + retryOffered: false }); + expect(withheld.body).not.toMatch(/retry/i); + expect(JSON.stringify(allRows(withheld.metadata))).not.toMatch(/retry/i); + expect(withheld.body).toContain("Paperclip is holding this task until its saved work is settled."); + } +}); diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index 65a321b34a9..48c7ecded67 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -27,6 +27,13 @@ export type StrandedRecoveryNoticeSeed = { nextAction?: string; }; +export const SAVED_WORK_HELD_SENTENCE = "Paperclip is holding this task until its saved work is settled."; + +export function withoutRetryInstruction(text: string): string { + const kept = text.split(/(?<=[.!?])\s+/).filter(sentence => !/\bretry\b/i.test(sentence)); + return [...kept, SAVED_WORK_HELD_SENTENCE].join(" "); +} + export type StrandedRecoveryEscalationNotice = { body: string; presentation: IssueCommentPresentation; @@ -270,6 +277,7 @@ export function buildStrandedRecoveryEscalationNotice(input: { recoveryCause?: string | null; recoveryActionId: string; continuesAutomatically?: boolean; + retryOffered?: boolean; recoveryOwner: { id: string; name: string | null } | null | undefined; sourceRun: { id: string; @@ -292,7 +300,8 @@ export function buildStrandedRecoveryEscalationNotice(input: { tone: "danger" as const, } : SETUP_FAILURE_NOTICES[input.sourceRun?.errorCode ?? ""] ?? input.seed; const fallbackBody = input.fallbackBody?.trim(); - const body = seed?.body ?? (fallbackBody || DEFAULT_STRANDED_RECOVERY_NOTICE_BODY); + const honest = (text: string) => input.retryOffered === false ? withoutRetryInstruction(text) : text; + const body = honest(seed?.body ?? (fallbackBody || DEFAULT_STRANDED_RECOVERY_NOTICE_BODY)); const title = STRANDED_RECOVERY_NOTICE_TITLES_BY_RUN_ERROR_CODE[input.sourceRun?.errorCode?.trim() ?? ""] ?? seed?.title ?? @@ -310,9 +319,9 @@ export function buildStrandedRecoveryEscalationNotice(input: { ), keyValueRow( "Next action", - seed?.nextAction ?? (input.recoveryOwner + honest(seed?.nextAction ?? (input.recoveryOwner ? "The assigned agent continues the task from the saved work" - : RETRY_FROM_SAVED_WORK_NEXT_ACTION), + : RETRY_FROM_SAVED_WORK_NEXT_ACTION)), ), ]; diff --git a/server/src/services/recovery/task-automatic-attempts.ts b/server/src/services/recovery/task-automatic-attempts.ts index 20a2aa266f7..4845f36b8dc 100644 --- a/server/src/services/recovery/task-automatic-attempts.ts +++ b/server/src/services/recovery/task-automatic-attempts.ts @@ -1,6 +1,6 @@ import { and, desc, eq, gte, inArray, sql } from "drizzle-orm"; import { activityLog, heartbeatRuns, type Db } from "@paperclipai/db"; -import { ASSIGNED_WORK_CONTINUED_ACTION, AUTOMATIC_RETRY_REASONS, SCHEDULER_TRANSIENT_RETRY_REASON, freshContinuationTimes } from "./continuation-ledger.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION, AUTOMATIC_RETRY_REASONS, SCHEDULER_TRANSIENT_RETRY_REASON, freshContinuationTimes, startedOrPendingAutomaticRetry } from "./continuation-ledger.js"; import { CONTINUATION_CAP, CONTINUATION_CAP_WINDOW_MS } from "./assigned-work-invariant.js"; export { ASSIGNED_WORK_CONTINUED_ACTION, SCHEDULER_TRANSIENT_RETRY_REASON }; @@ -14,7 +14,8 @@ export async function taskAutomaticAttempts(db: Db, companyId: string, issueId: .orderBy(desc(activityLog.createdAt)).limit(64), db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since))) + inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since), + startedOrPendingAutomaticRetry())) .orderBy(desc(heartbeatRuns.createdAt)).limit(64), ]); const fresh = (await freshContinuationTimes(db, companyId, issueId, since))