diff --git a/server/src/__tests__/failed-continuation-continues-again.test.ts b/server/src/__tests__/failed-continuation-continues-again.test.ts index ba7caebdcd2..22decb5960a 100644 --- a/server/src/__tests__/failed-continuation-continues-again.test.ts +++ b/server/src/__tests__/failed-continuation-continues-again.test.ts @@ -3,10 +3,11 @@ import { and, eq, inArray, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { activityLog, agentWakeupRequests, agents, companies, createDb, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; import { recoveryService } from "../services/recovery/service.js"; -import { ASSIGNED_WORK_CONTINUED_ACTION, assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION, ASSIGNED_WORK_REPAIRS_PER_MINUTE, assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; +import { taskAutomaticAttempts, transientRetryWithinTaskCap } from "../services/recovery/task-automatic-attempts.js"; import { FRESH_CONTINUATION_SCHEMA, continueHeldExecutions, ensureFreshContinuation, resetBacklogContinuationPacingForTests } from "../services/fresh-continuation-fallback.js"; import { PERMANENT_FAILURE_CODES, PERSON_OWNED_FAILURE_CODES } from "../services/continuation-cause.js"; -import { AUTOMATIC_RETRY_REASONS, SHARED_CONTINUATION_WINDOW_MS } from "../services/recovery/continuation-ledger.js"; +import { AUTOMATIC_RETRY_REASONS, SHARED_CONTINUATION_WINDOW_MS, sharedContinuationAttempts } from "../services/recovery/continuation-ledger.js"; import { SERVER_RESTART_RETRY_MAX_ATTEMPTS, SERVER_RESTART_RETRY_REASON, SERVER_RESTART_RETRY_WINDOW_MS } from "../services/execution-recovery-attempt.js"; import { FRESH_CONTINUATIONS_PER_HOUR } from "../services/fresh-continuation-fallback.js"; import { createSystemRecovery } from "../services/system-recovery-state.js"; @@ -414,4 +415,102 @@ describePostgres("a continuation that fails before doing any work is continued a expect(await sweep.checkIssue({ companyId: f.companyId, issueId: task.id })).toMatchObject({ outcome: "repaired" }); expect(await continuationsOf(stopped.id)).toHaveLength(1); }, 60_000); + + async function blockedBehindRefusedRun(f: Awaited>) { + const blockedAt = new Date(Date.now() - 20 * MINUTE); + const task = await f.task({ status: "blocked", blockedTransitionAt: blockedAt, statusVersion: 3 }); + const source = await f.run(task.id, { errorCode: "adapter_failed", error: "native_instruction_materialization_handoff_unavailable", + finishedAt: new Date(blockedAt.getTime() - 500), resultJson: { workspaceRestoreFailure: "restore_failed" }, + contextSnapshot: { wakeReason: "issue_children_completed" } }); + const [hold] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: task.id, kind: "active_run_watchdog", + status: "active", ownerType: "board", returnOwnerAgentId: f.agentId, cause: "legacy_execution_requires_reconciliation", + fingerprint: `legacy-execution:${source.id}`, nextAction: "Held.", createdAt: blockedAt, + evidence: { runId: source.id, originalFailureCode: "adapter_failed", workspaceRestoreFailure: "restore_failed", + systemRecovery: { ...createSystemRecovery(source.id), state: "complete" } } }).returning(); + const refused = await f.run(task.id, { status: "cancelled", errorCode: "execution_reconciliation_required", startedAt: null, + retryOfRunId: source.id, finishedAt: new Date(blockedAt.getTime() + 1_000), createdAt: new Date(blockedAt.getTime() - 100), resultJson: {}, + contextSnapshot: { wakeReason: "issue_continuation_needed", retryReason: "issue_continuation_needed" } }); + return { task, source, hold: hold!, refused }; + } + + it("the sweep continues a blocked task whose system retry was refused at claim behind a recovered saved-work hold", async () => { + resetBacklogContinuationPacingForTests(db); + const f = await seed(); + const { task, source, hold, refused } = await blockedBehindRefusedRun(f); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id)).toContain(task.id); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id })) + .toMatchObject({ outcome: "repaired", verdict: { kind: "violation" } }); + const [record] = await continuationsOf(source.id); + expect(record?.id).toBe(hold.id); + expect(record?.evidence).toMatchObject({ continuationDelivery: "pending", freshContinuation: { sourceRunId: source.id, trigger: "assigned_work" }, + freshContinuationSettlement: { refusedAtDispatchRunId: refused.id, requestedBy: "system" } }); + expect((await taskRow(task.id)).status).toBe("todo"); + expect(queue.calls).toHaveLength(0); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id })) + .toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent" } }); + }, 60_000); + + it("a continuation the sweep records behind a refused run counts once toward the task's hourly cap", async () => { + resetBacklogContinuationPacingForTests(db); + const f = await seed(); + const { task } = await blockedBehindRefusedRun(f); + for (const minutesAgo of [50, 45, 40, 35, 30]) await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", + actorId: "system", action: ASSIGNED_WORK_CONTINUED_ACTION, entityType: "issue", entityId: task.id, + details: { outcome: "queued" }, createdAt: new Date(Date.now() - minutesAgo * MINUTE) }); + expect(await transientRetryWithinTaskCap(db, f.companyId, task.id, new Date())).toBe(true); + const sweep = assignedWorkSweep(db, { enqueueWakeup: wakeQueue().enqueueWakeup }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: task.id })).toMatchObject({ outcome: "repaired" }); + const attempts = await taskAutomaticAttempts(db, f.companyId, task.id, new Date()); + expect(attempts.started).toHaveLength(FRESH_CONTINUATIONS_PER_HOUR); + expect(await transientRetryWithinTaskCap(db, f.companyId, task.id, new Date())).toBe(false); + }, 60_000); + + it("continuations behind refused runs spend the sweep's per-minute repair budget", async () => { + resetBacklogContinuationPacingForTests(db); + const f = await seed(); + const tasks = []; + for (let index = 0; index < 20; index++) tasks.push((await blockedBehindRefusedRun(f)).task); + let clock = new Date(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: wakeQueue().enqueueWakeup, now: () => clock }); + const first = []; + for (const task of tasks) first.push(await sweep.checkIssue({ companyId: f.companyId, issueId: task.id })); + expect(first.filter(check => check?.outcome === "repaired")).toHaveLength(ASSIGNED_WORK_REPAIRS_PER_MINUTE); + const deferred = first.filter(check => check?.outcome === "deferred_rate_limit"); + expect(deferred).toHaveLength(20 - ASSIGNED_WORK_REPAIRS_PER_MINUTE); + expect(deferred.every(check => check!.retryAt!.getTime() > clock.getTime())).toBe(true); + clock = new Date(clock.getTime() + 66_000); + const second = []; + for (const task of tasks) second.push(await sweep.checkIssue({ companyId: f.companyId, issueId: task.id })); + expect(second.filter(check => check?.outcome === "repaired")).toHaveLength(20 - ASSIGNED_WORK_REPAIRS_PER_MINUTE); + }, 120_000); + + it("a sweep continuation interrupted right after it was recorded still counts once", async () => { + resetBacklogContinuationPacingForTests(db); + const f = await seed(); + const task = await f.task(); + const failed = await f.continued(task.id); + for (const minutesAgo of [50, 45, 40, 35]) await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", + actorId: "system", action: ASSIGNED_WORK_CONTINUED_ACTION, entityType: "issue", entityId: task.id, + details: { outcome: "queued" }, createdAt: new Date(Date.now() - minutesAgo * MINUTE) }); + const before = (await taskAutomaticAttempts(db, f.companyId, task.id, new Date())).started.length; + const crashAfterCommit = new Proxy(db, { get(target, property) { + if (property === "update") return (table: unknown) => { + if (table === activityLog) throw new Error("server stopped"); + return target.update(table as typeof activityLog); + }; + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + } }); + const sweep = assignedWorkSweep(crashAfterCommit, { enqueueWakeup: wakeQueue().enqueueWakeup }); + await expect(sweep.checkIssue({ companyId: f.companyId, issueId: task.id })).rejects.toThrow("server stopped"); + expect(await continuationsOf(failed.id)).toHaveLength(1); + const attempts = await taskAutomaticAttempts(db, f.companyId, task.id, new Date()); + expect(attempts.started).toHaveLength(before + 1); + expect(attempts.started).toHaveLength(5); + expect(await sharedContinuationAttempts(db, f.companyId, task.id, new Date())).toHaveLength(before + 1); + expect(await transientRetryWithinTaskCap(db, f.companyId, task.id, new Date())).toBe(true); + }, 60_000); }); diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index e4f65754f30..0fb268e48ae 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -28,6 +28,8 @@ import { createPostgresRunDispatchAdapter } from "../modules/run-dispatch/adapte import { createSystemRecovery } from "./system-recovery-state.js"; import { requestTaskRetry } from "./task-retry.js"; import { terminalizeLegacyExecution } from "./legacy-execution-recovery.js"; +import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js"; +import { taskAutomaticAttempts } from "./recovery/task-automatic-attempts.js"; const support = await getEmbeddedPostgresTestSupport(); const suite = support.supported ? describe : describe.skip; @@ -1351,4 +1353,125 @@ suite("fresh-continuation fallback", () => { const [next] = await successors(f.runId); expect(await readFreshWorkspaceReadiness(db, next!)).toMatchObject({ mode: "fresh_isolated_workspace" }); }, 60_000); + + async function followUpFailedBeforeProviderWork(options: { restoreFailed?: boolean } = {}) { + const issueId = randomUUID(), runId = randomUUID(); + const endedAt = new Date(Date.now() - 60_000); + await db.insert(issues).values({ id: issueId, companyId, title: "Summarize the subtasks", status: "in_progress", assigneeAgentId: agentId }); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issueId, mode: "isolated", + strategyType: "git_worktree", name: "Task workspace", cwd: "/work/remote-task" }).returning(); + await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issueId)); + const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "automation", issueId, + runtimeMode: "legacy", status: "running", startedAt: new Date(endedAt.getTime() - 10_000), + createdAt: new Date(endedAt.getTime() - 60_000), responsibleUserId: userId, + contextSnapshot: { issueId, executionWorkspaceId: workspace!.id, wakeReason: "issue_children_completed" } }).returning(); + const [environment] = await db.insert(environments).values({ name: `sandbox-${runId}`, driver: "sandbox" }).returning(); + await db.insert(environmentLeases).values({ companyId, issueId, executionWorkspaceId: workspace!.id, + heartbeatRunId: runId, environmentId: environment!.id, status: "active", leasePolicy: "reuse_by_environment", provider: "daytona", + providerLeaseId: `sandbox-${runId}`, metadata: { remoteCwd: "/work/remote-task", driver: "sandbox", + workspaceRealization: { authoritativeRoot: "/work/remote-task" } } }); + await terminalizeLegacyExecution({ db, run: run!, status: "failed", patch: { errorCode: "adapter_failed", + error: "native_instruction_materialization_handoff_unavailable", finishedAt: endedAt, + resultJson: options.restoreFailed === false ? { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } + : { workspaceRestoreFailure: "restore_failed" } } }); + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + return { issueId, runId, actionId: hold?.id ?? null }; + } + + async function savedWorkRecoveryState(actionId: string, runId: string, state: "running" | "complete") { + await db.update(issueRecoveryActions).set({ updatedAt: new Date(), + evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ systemRecovery: { ...createSystemRecovery(runId), state } })}::jsonb` }) + .where(eq(issueRecoveryActions.id, actionId)); + } + + async function systemRetryAtClaim(f: { issueId: string; runId: string }) { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, + runtimeMode: "legacy", status: "queued", retryOfRunId: f.runId, + contextSnapshot: { issueId: f.issueId, wakeReason: "issue_continuation_needed", retryReason: "issue_continuation_needed", + retryOfRunId: f.runId } }).returning(); + const outcome = await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId, runId: run!.id, + expectedStatus: "queued", now: new Date() }); + return { run: run!, outcome }; + } + + async function deliveredContinuationIsClaimable(f: { issueId: string; runId: string; actionId: string | null }, refusedRunId: string) { + await heartbeatService(db).reconcileStrandedAssignedIssues({ companyId, issueId: f.issueId }); + expect((await issue(f.issueId)).status).not.toBe("blocked"); + expect(await getExecutionBlocker(db, companyId, f.issueId)).toBeNull(); + await occupySlot(); + await deliver(); + const next = (await successors(f.runId)).filter(run => run.id !== refusedRunId); + expect(next).toHaveLength(1); + expect(next[0]!.status).toBe("queued"); + expect(await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId, runId: next[0]!.id, + expectedStatus: "queued", now: new Date() })).toEqual({ outcome: "not_stale" }); + return next[0]!; + } + + it("continues a follow-up that failed before provider work with unrestored files, when the saved work was recovered before the system retry was refused at claim", async () => { + const f = await followUpFailedBeforeProviderWork(); + expect(f.actionId).not.toBeNull(); + await savedWorkRecoveryState(f.actionId!, f.runId, "complete"); + const refused = await systemRetryAtClaim(f); + expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" }); + expect((await holdRow(f.actionId!)).evidence).toMatchObject({ + freshContinuation: { sourceRunId: f.runId }, continuationDelivery: "pending", + freshContinuationSettlement: { refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } }); + await deliveredContinuationIsClaimable(f, refused.run.id); + }, 60_000); + + it("continues a follow-up that failed before provider work with unrestored files, when the saved work was recovered after the system retry was refused at claim", async () => { + const f = await followUpFailedBeforeProviderWork(); + await savedWorkRecoveryState(f.actionId!, f.runId, "running"); + const refused = await systemRetryAtClaim(f); + expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" }); + expect((await holdRow(f.actionId!)).evidence.freshContinuationCheck) + .toMatchObject({ kind: "exact_recovery_pending", reason: "saved_work_recovery_running" }); + await savedWorkRecoveryState(f.actionId!, f.runId, "complete"); + await makeDue(f.actionId!); + expect(await continueHeldExecutions(db)).toMatchObject({ continued: 1 }); + expect((await holdRow(f.actionId!)).evidence).toMatchObject({ freshContinuation: { sourceRunId: f.runId }, + freshContinuationSettlement: { refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } }); + await deliveredContinuationIsClaimable(f, refused.run.id); + }, 60_000); + + it("continues a follow-up that failed before provider work with its files in place, after its system retry was refused at claim", async () => { + const f = await followUpFailedBeforeProviderWork({ restoreFailed: false }); + const refused = await systemRetryAtClaim(f); + expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" }); + expect((await holdRow(f.actionId!)).evidence).toMatchObject({ freshContinuation: { sourceRunId: f.runId }, + freshContinuationSettlement: { refusedAtDispatchRunId: refused.run.id, requestedBy: "system" } }); + await deliveredContinuationIsClaimable(f, refused.run.id); + }, 60_000); + + it("counts only automatic retries that started, or still may, against the task's hourly budget", async () => { + const f = await followUpFailedBeforeProviderWork(); + for (let index = 0; index < FRESH_CONTINUATIONS_PER_HOUR; index++) { + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, runtimeMode: "legacy", + status: "cancelled", errorCode: "execution_reconciliation_required", finishedAt: new Date(), retryOfRunId: f.runId, + scheduledRetryReason: "transient_failure", contextSnapshot: { issueId: f.issueId, wakeReason: "transient_failure_retry" } }); + } + expect(await sharedContinuationAttempts(db, companyId, f.issueId, new Date())).toHaveLength(0); + expect((await taskAutomaticAttempts(db, companyId, f.issueId, new Date())).started).toHaveLength(0); + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, runtimeMode: "legacy", + status: "failed", errorCode: "adapter_failed", startedAt: new Date(), finishedAt: new Date(), retryOfRunId: f.runId, + scheduledRetryReason: "transient_failure", contextSnapshot: { issueId: f.issueId, wakeReason: "transient_failure_retry" } }); + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", issueId: f.issueId, runtimeMode: "legacy", + status: "scheduled_retry", retryOfRunId: f.runId, + scheduledRetryReason: "transient_failure", contextSnapshot: { issueId: f.issueId, wakeReason: "transient_failure_retry" } }); + expect(await sharedContinuationAttempts(db, companyId, f.issueId, new Date())).toHaveLength(2); + expect((await taskAutomaticAttempts(db, companyId, f.issueId, new Date())).started).toHaveLength(2); + }, 60_000); + + it("still leaves a person's failed Retry of a follow-up for that person after its successor is refused at claim", async () => { + const f = await followUpFailedBeforeProviderWork(); + await savedWorkRecoveryState(f.actionId!, f.runId, "complete"); + await db.update(heartbeatRuns).set({ contextSnapshot: sql`${heartbeatRuns.contextSnapshot} || '{"wakeReason":"task_retry"}'::jsonb` }) + .where(eq(heartbeatRuns.id, f.runId)); + const refused = await systemRetryAtClaim(f); + expect(refused.outcome).toMatchObject({ outcome: "cancelled", errorCode: "execution_reconciliation_required" }); + expect((await holdRow(f.actionId!)).evidence.freshContinuation).toBeUndefined(); + expect(await ensureFreshContinuation(db, { companyId, runId: f.runId, trigger: "held_recovery" })) + .toEqual({ kind: "person_owned", reason: "retry_needed" }); + }, 60_000); }); diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 39cb5d8d485..f8ad3bb99d0 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -26,7 +26,7 @@ import { notifyRoutineExecutionStatus } from "./routine-execution-status-notific import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { readRunCancellation } from "./run-cancellation.js"; import { readSystemRecovery } from "./system-recovery-state.js"; -import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuesAutomaticallySql, isPersonOwnedFailureCode } from "./continuation-cause.js"; +import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuationCauseIsSystemOwnedSql, continuesAutomaticallySql, isPersonOwnedFailureCode, recoveryRunSql } from "./continuation-cause.js"; import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js"; export { AUTOMATIC_CONTINUATION_WAKE_REASONS, PERSON_OWNED_FAILURE_CODES, isPersonOwnedFailureCode, isRecoveryRun } from "./continuation-cause.js"; @@ -813,7 +813,10 @@ function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQ and retry.run_id is null and retry.payload->>'issueId' = ${input.issueId}::text and retry.requested_at > source.finished_at) or exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and held.context_snapshot->>'wakeReason' = 'task_retry' and held.finished_at > ${since}))`); - const recent = source(sql`${continuesAutomaticallySql("source")} + const systemRetriedSource = sql`(${continuationCauseIsSystemOwnedSql("source")} + and coalesce(source.context_snapshot->>'wakeReason', '') <> 'task_retry' and not coalesce(source.context_snapshot ? 'explicitUserContinuation', false) + and exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and ${recoveryRunSql("held")}))`; + const recent = source(sql`(${continuesAutomaticallySql("source")} or ${systemRetriedSource}) and greatest(source.finished_at, (select max(held.finished_at) from ${heartbeatRuns} held where ${refusedLater})) > ${since}`); return { recent, personAsked }; } diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index c7ac8ea5525..5559500dc9f 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -38,8 +38,7 @@ import { attributeBlock, BLOCK_ATTRIBUTION_WINDOW_MS, blockedStatusDetailsSql } import { ASSIGNED_WORK_CONTINUED_ACTION, PAUSED_FOR_AUTOMATIC_WORK, taskAutomaticAttempts } from "./task-automatic-attempts.js"; import { automaticRunCreationHeldUntil, readAutomaticWorkStatus, reserveAutomaticRunCreation } from "./automatic-run-brake.js"; import { isReattachPhaseActive, reattachPhaseRecheckAt } from "../heartbeat/run-admission.js"; -import { ensureFreshContinuation, type FreshContinuationResult } from "../fresh-continuation-fallback.js"; -import { canContinueRecoveryRunAutomatically } from "../continuation-cause.js"; +import { continueSweepTaskFromSavedWork } from "./sweep-saved-work-continuation.js"; import { STRANDED_CONTINUATION_MAX_AGE_MS, STRANDED_KIND, strandedHoldContinuesAutomatically, type StrandedHoldRun } from "./stranded-hold-continuation.js"; import { continuationActivityMessage, @@ -499,6 +498,13 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { * Chat conversations and blocks older than the stale window are left out: * the first wait on a person's message, the second is only reported. */ + function latestRunRefusedAtDispatch(at: Date) { + return sql`coalesce((select lr.status = 'cancelled' and lr.started_at is null and lr.error_code = 'execution_reconciliation_required' + and lr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz + from heartbeat_runs lr where lr.company_id = ${issues.companyId} and (lr.context_snapshot->>'issueId') = ${issues.id}::text + order by lr.created_at desc limit 1), false)`; + } + async function listSuspects(scope: { companyId?: string; issueId?: string; createdAfter?: Date | null }, cursor: string | null, limit: number) { const at = now(); const liveRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} @@ -559,7 +565,9 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { -- except a recent board escalation, which the predicate judges by its failed run's cause and not (ra.kind = ${STRANDED_KIND} and ra.owner_type = 'board' and exists (select 1 from heartbeat_runs sr where sr.company_id = ra.company_id and sr.id::text = ra.evidence->>'latestRunId' - and sr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz)))`, + and sr.finished_at > ${new Date(at.getTime() - STRANDED_CONTINUATION_MAX_AGE_MS).toISOString()}::timestamptz)) + and not (${issues.status} = 'blocked' and ra.cause not in ('configuration_incomplete', 'ownership_owner_unprovable') + and ${latestRunRefusedAtDispatch(at)}))`, // An assignee that cannot take work (paused, being set up, held) is // its own wait; the predicate names it when anything else asks. sql`exists (select 1 from agents a where a.id = ${issues.assigneeAgentId} and a.company_id = ${issues.companyId} @@ -602,6 +610,14 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { // hold's owner (the fresh-continuation fallback, same per-task cap). The // sweep reports it and repairs only blocks that have no hold at all, so if (issue.status === "blocked" && facts.activeRecoveryAction && !facts.activeRecoveryAction.continuedBySweep) { + const behindRefusedRun = facts.latestRun?.status === "cancelled" && facts.latestRun.errorCode === "execution_reconciliation_required"; + const deferred = behindRefusedRun ? repairBudgetDeferral(at) : null; + if (deferred) return { verdict, outcome: "deferred_rate_limit", retryAt: deferred }; + const refused = behindRefusedRun ? await continueFromSavedWork(issue, facts, at) : null; + if (refused?.kind === "continued") repairTimes.push(at.getTime()); + if (refused?.kind === "continued" || refused?.kind === "already_continued") return { verdict, outcome: "repaired", retryAt: null }; + if (refused?.kind === "backoff" || refused?.kind === "exact_recovery_pending") + return { verdict, outcome: refused.kind === "backoff" ? "deferred_cap" : "waiting_on_recovery", retryAt: refused.retryAt }; return { verdict, outcome: "left_to_hold_owner", retryAt: null }; } // A stalled native run is ended by native recovery, not here: the sweep @@ -623,11 +639,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { } // A small process-wide budget keeps a post-deploy backlog from starting // hundreds of runs at once. Deferred tasks come back on their deadline. - const minuteAgo = at.getTime() - 60_000; - while (repairTimes.length && repairTimes[0] <= minuteAgo) repairTimes.shift(); - if (repairTimes.length >= ASSIGNED_WORK_REPAIRS_PER_MINUTE) { - return { verdict, outcome: "deferred_rate_limit", retryAt: new Date(repairTimes[0] + 60_000 + Math.floor(Math.random() * 5_000)) }; - } + const budgetDeferral = repairBudgetDeferral(at); + if (budgetDeferral) return { verdict, outcome: "deferred_rate_limit", retryAt: budgetDeferral }; const heldUntil = facts.latestRun ? await automaticRunCreationHeldUntil(db, issue.companyId, { issueId: issue.id, sourceRunId: facts.latestRun.id }) : null; if (heldUntil) return { verdict, outcome: "deferred_rate_limit", retryAt: heldUntil }; @@ -642,7 +655,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { await settleContinuationActivity(claimed, claimed.reason, "restored_to_review"); return { verdict: claimed.verdict, outcome: "repaired", retryAt: null }; } - const fresh = await continueFromSavedWork(issue, facts, at); + const fresh = await continueFromSavedWork(issue, facts, at, claimed); if (fresh?.kind === "continued") { await settleContinuationActivity(claimed, claimed.reason, "queued", { continuedFromSavedWork: "true" }); return { verdict: claimed.verdict, outcome: "repaired", retryAt: claimed.nextCheckAt }; @@ -683,6 +696,13 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { return { verdict: claimed.verdict, outcome: run ? "repaired" : "enqueue_refused", retryAt: claimed.nextCheckAt }; } + function repairBudgetDeferral(at: Date): Date | null { + const minuteAgo = at.getTime() - 60_000; + while (repairTimes.length && repairTimes[0] <= minuteAgo) repairTimes.shift(); + return repairTimes.length >= ASSIGNED_WORK_REPAIRS_PER_MINUTE + ? new Date(repairTimes[0] + 60_000 + Math.floor(Math.random() * 5_000)) : null; + } + function reportStalledRun(issue: IssueRow, runId: string, stall: NonNullable | null, at: Date) { if (reportedStalledRuns.has(runId)) return; if (reportedStalledRuns.size >= 1_000) reportedStalledRuns.clear(); @@ -725,20 +745,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { return { verdict: claimed.verdict, outcome: run ? "replaced_stalled_run" : "enqueue_refused", retryAt: claimed.nextCheckAt }; } - async function continueFromSavedWork(issue: IssueRow, facts: AssignedWorkFacts, at: Date): Promise { - const source = facts.latestRun; - if (!source?.finishedAt || !["failed", "timed_out", "interrupted", "cancelled"].includes(source.status) - || (source.status === "cancelled" && source.errorCode === "execution_reconciliation_required")) return null; - const [run] = await db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode, error: heartbeatRuns.error, - retryOfRunId: heartbeatRuns.retryOfRunId, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, - contextSnapshot: heartbeatRuns.contextSnapshot, runnerProfileJson: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns) - .where(and(eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.id, source.id))); - if (!run || !canContinueRecoveryRunAutomatically(run)) return null; - return ensureFreshContinuation(db, { companyId: issue.companyId, runId: source.id, trigger: "assigned_work", now: at }) - .catch(error => { - logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id }, "assigned-work continuation from saved work deferred"); - return null; - }); + function continueFromSavedWork(issue: IssueRow, facts: AssignedWorkFacts, at: Date, claim?: { activityId: string; reason: ViolationReason }) { + return continueSweepTaskFromSavedWork(db, { companyId: issue.companyId, issueId: issue.id, latestRun: facts.latestRun }, at, claim); } async function startSavedWorkRecovery(issue: IssueRow, facts: AssignedWorkFacts, diff --git a/server/src/services/recovery/continuation-ledger.ts b/server/src/services/recovery/continuation-ledger.ts index 995f1eed742..5b7ecb5a861 100644 --- a/server/src/services/recovery/continuation-ledger.ts +++ b/server/src/services/recovery/continuation-ledger.ts @@ -6,6 +6,11 @@ export const SCHEDULER_TRANSIENT_RETRY_REASON = "transient_failure"; export const AUTOMATIC_RETRY_REASONS = [SCHEDULER_TRANSIENT_RETRY_REASON, "server_restart", "interrupted_preparation"]; export const SHARED_CONTINUATION_WINDOW_MS = 60 * 60_000; +export function startedOrPendingAutomaticRetry() { + return sql`not (${heartbeatRuns.status} = 'cancelled' and ${heartbeatRuns.startedAt} is null + and coalesce(${heartbeatRuns.errorCode}, '') = 'execution_reconciliation_required')`; +} + export async function freshContinuationTimes(db: Db, companyId: string, issueId: string, since: Date) { const rows = await db.select({ recordedAt: sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}'`, trigger: sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,trigger}'` }).from(issueRecoveryActions).where(and( @@ -25,7 +30,8 @@ export async function sharedContinuationAttempts(db: Db, companyId: string, issu sql`coalesce(${activityLog.details}->>'continuedFromSavedWork', '') <> 'true'`)).limit(64), db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since))).limit(64), + inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since), + startedOrPendingAutomaticRetry())).limit(64), freshContinuationTimes(db, companyId, issueId, since), ]); return [...plainContinuations.map(row => row.createdAt), ...retries.map(row => row.createdAt), ...fresh.map(record => record.at)] diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 8b9a3ad4a5f..96ba761bb7e 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -176,6 +176,7 @@ import { buildExecutionReviewParticipantRecoveryNoticeSeed, buildExecutionReviewParticipantUnavailableNoticeSeed, buildStrandedRecoveryEscalationNotice, + withoutRetryInstruction, buildNoRunningExecutionNoticeSeed, RETRY_FROM_SAVED_WORK_NEXT_ACTION, type StrandedRecoveryNoticeSeed, @@ -185,6 +186,7 @@ import { isStrandedIssueRecoveryOriginKind, } from "./origins.js"; import { withRecoveryContext } from "./status-only-context.js"; +import { getExecutionBlocker } from "../execution-blocker.js"; import { currentRunFailureText } from "../native-runtime/native-exact-resume-failure.js"; import { isAutomaticRecoverySuppressedByPauseHold } from "./pause-hold-guard.js"; import { rehomeStrandedLegacyWorkspaceTasks } from "./legacy-workspace-rehome.js"; @@ -4251,6 +4253,13 @@ export function recoveryService( recoveryCause, successfulRunHandoffEvidence: input.successfulRunHandoffEvidence, }, tx); + const executionBlocker = await getExecutionBlocker(tx as unknown as Db, input.issue.companyId, input.issue.id); + const retryOffered = !(executionBlocker && executionBlocker.canRetry === false); + if (!retryOffered && /\bretry\b/i.test(recoveryAction.nextAction ?? "")) { + recoveryAction.nextAction = withoutRetryInstruction(recoveryAction.nextAction!); + await tx.update(issueRecoveryActions).set({ nextAction: recoveryAction.nextAction }) + .where(and(eq(issueRecoveryActions.companyId, input.issue.companyId), eq(issueRecoveryActions.id, recoveryAction.id))); + } const isProviderQuotaWait = recoveryCause === "provider_quota" && !recoveryAction.ownerAgentId && @@ -4334,6 +4343,7 @@ export function recoveryService( fallbackBody: input.comment, recoveryCause, recoveryActionId: recoveryAction.id, + retryOffered, continuesAutomatically: Boolean(input.latestRun) && recoveryAction.kind === STRANDED_KIND && recoveryAction.ownerType === "board" && input.latestRun?.agentId === input.issue.assigneeAgentId && strandedRunCauseIsTemporary({ diff --git a/server/src/services/recovery/stranded-notice.test.ts b/server/src/services/recovery/stranded-notice.test.ts index 6c6e932714d..9ed9e3d9eb6 100644 --- a/server/src/services/recovery/stranded-notice.test.ts +++ b/server/src/services/recovery/stranded-notice.test.ts @@ -401,3 +401,16 @@ it("tells the operator to switch a preconfigured Runner home to an isolated one" expect(notice.nextAction).toContain("isolated Runner home"); expect(notice.nextAction).not.toContain("Choose an AI connection"); }); + +it("never tells a person to press Retry when the task offers no Retry", () => { + const sourceRun = { id: "run-1", status: "cancelled", errorCode: "execution_reconciliation_required" }; + for (const seed of [buildNoRunningExecutionNoticeSeed({ status: "in_progress" }), null]) { + const offered = buildStrandedRecoveryEscalationNotice({ seed, recoveryActionId: "action-1", recoveryOwner: null, sourceRun }); + expect(`${offered.body} ${JSON.stringify(allRows(offered.metadata))}`).toMatch(/Press Retry/); + const withheld = buildStrandedRecoveryEscalationNotice({ seed, recoveryActionId: "action-1", recoveryOwner: null, sourceRun, + retryOffered: false }); + expect(withheld.body).not.toMatch(/retry/i); + expect(JSON.stringify(allRows(withheld.metadata))).not.toMatch(/retry/i); + expect(withheld.body).toContain("Paperclip is holding this task until its saved work is settled."); + } +}); diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index 65a321b34a9..48c7ecded67 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -27,6 +27,13 @@ export type StrandedRecoveryNoticeSeed = { nextAction?: string; }; +export const SAVED_WORK_HELD_SENTENCE = "Paperclip is holding this task until its saved work is settled."; + +export function withoutRetryInstruction(text: string): string { + const kept = text.split(/(?<=[.!?])\s+/).filter(sentence => !/\bretry\b/i.test(sentence)); + return [...kept, SAVED_WORK_HELD_SENTENCE].join(" "); +} + export type StrandedRecoveryEscalationNotice = { body: string; presentation: IssueCommentPresentation; @@ -270,6 +277,7 @@ export function buildStrandedRecoveryEscalationNotice(input: { recoveryCause?: string | null; recoveryActionId: string; continuesAutomatically?: boolean; + retryOffered?: boolean; recoveryOwner: { id: string; name: string | null } | null | undefined; sourceRun: { id: string; @@ -292,7 +300,8 @@ export function buildStrandedRecoveryEscalationNotice(input: { tone: "danger" as const, } : SETUP_FAILURE_NOTICES[input.sourceRun?.errorCode ?? ""] ?? input.seed; const fallbackBody = input.fallbackBody?.trim(); - const body = seed?.body ?? (fallbackBody || DEFAULT_STRANDED_RECOVERY_NOTICE_BODY); + const honest = (text: string) => input.retryOffered === false ? withoutRetryInstruction(text) : text; + const body = honest(seed?.body ?? (fallbackBody || DEFAULT_STRANDED_RECOVERY_NOTICE_BODY)); const title = STRANDED_RECOVERY_NOTICE_TITLES_BY_RUN_ERROR_CODE[input.sourceRun?.errorCode?.trim() ?? ""] ?? seed?.title ?? @@ -310,9 +319,9 @@ export function buildStrandedRecoveryEscalationNotice(input: { ), keyValueRow( "Next action", - seed?.nextAction ?? (input.recoveryOwner + honest(seed?.nextAction ?? (input.recoveryOwner ? "The assigned agent continues the task from the saved work" - : RETRY_FROM_SAVED_WORK_NEXT_ACTION), + : RETRY_FROM_SAVED_WORK_NEXT_ACTION)), ), ]; diff --git a/server/src/services/recovery/sweep-saved-work-continuation.ts b/server/src/services/recovery/sweep-saved-work-continuation.ts new file mode 100644 index 00000000000..66e354f6c6d --- /dev/null +++ b/server/src/services/recovery/sweep-saved-work-continuation.ts @@ -0,0 +1,51 @@ +import { and, desc, eq, inArray, sql } from "drizzle-orm"; +import { activityLog, heartbeatRuns, type Db } from "@paperclipai/db"; +import { logger } from "../../middleware/logger.js"; +import { continueFreshInTransaction, type FreshContinuationResult } from "../fresh-continuation-fallback.js"; +import { canContinueRecoveryRunAutomatically } from "../continuation-cause.js"; +import { continuationActivityMessage, type AssignedWorkFacts, type ViolationReason } from "./assigned-work-invariant.js"; + +type SweepTask = { companyId: string; issueId: string; latestRun: AssignedWorkFacts["latestRun"] }; +type SweepClaim = { activityId: string; reason: ViolationReason }; + +function continueAndSettleClaim(db: Db, task: SweepTask, runId: string, at: Date, claim?: SweepClaim) { + return db.transaction(async tx => { + const result = await continueFreshInTransaction(tx as unknown as Db, { companyId: task.companyId, runId, trigger: "assigned_work", now: at }); + if (result.kind === "continued" && claim) { + const update = { outcome: "queued", message: continuationActivityMessage(claim.reason), continuedFromSavedWork: "true" }; + await tx.update(activityLog).set({ details: sql`coalesce(${activityLog.details}, '{}'::jsonb) || ${JSON.stringify(update)}::jsonb` }) + .where(and(eq(activityLog.companyId, task.companyId), eq(activityLog.id, claim.activityId))); + } + return result; + }); +} + +/** The sweep's continuation of a task from saved work: from its failed latest run, or from the run a refused continuation was for. */ +export async function continueSweepTaskFromSavedWork(db: Db, task: SweepTask, at: Date, claim?: SweepClaim): Promise { + const source = task.latestRun; + if (!source?.finishedAt || !["failed", "timed_out", "interrupted", "cancelled"].includes(source.status)) return null; + if (source.status === "cancelled" && source.errorCode === "execution_reconciliation_required") { + const [refusedFor] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, task.companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${task.issueId}`, + inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]), sql`${heartbeatRuns.finishedAt} is not null`, + sql`not (${heartbeatRuns.status} = 'cancelled' and ${heartbeatRuns.startedAt} is null + and coalesce(${heartbeatRuns.errorCode}, '') = 'execution_reconciliation_required')`)) + .orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); + if (!refusedFor) return null; + return continueAndSettleClaim(db, task, refusedFor.id, at, claim) + .catch(error => { + logger.warn({ err: error, companyId: task.companyId, issueId: task.issueId }, "assigned-work continuation behind a refused run deferred"); + return null; + }); + } + const [run] = await db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode, error: heartbeatRuns.error, + retryOfRunId: heartbeatRuns.retryOfRunId, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, + contextSnapshot: heartbeatRuns.contextSnapshot, runnerProfileJson: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, task.companyId), eq(heartbeatRuns.id, source.id))); + if (!run || !canContinueRecoveryRunAutomatically(run)) return null; + return continueAndSettleClaim(db, task, source.id, at, claim) + .catch(error => { + logger.warn({ err: error, companyId: task.companyId, issueId: task.issueId }, "assigned-work continuation from saved work deferred"); + return null; + }); +} diff --git a/server/src/services/recovery/task-automatic-attempts.ts b/server/src/services/recovery/task-automatic-attempts.ts index e1e47d17673..82f5fcba619 100644 --- a/server/src/services/recovery/task-automatic-attempts.ts +++ b/server/src/services/recovery/task-automatic-attempts.ts @@ -1,6 +1,6 @@ import { and, desc, eq, gte, inArray, sql } from "drizzle-orm"; import { activityLog, heartbeatRuns, type Db } from "@paperclipai/db"; -import { ASSIGNED_WORK_CONTINUED_ACTION, AUTOMATIC_RETRY_REASONS, SCHEDULER_TRANSIENT_RETRY_REASON, freshContinuationTimes } from "./continuation-ledger.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION, AUTOMATIC_RETRY_REASONS, SCHEDULER_TRANSIENT_RETRY_REASON, freshContinuationTimes, startedOrPendingAutomaticRetry } from "./continuation-ledger.js"; import { CONTINUATION_CAP, CONTINUATION_CAP_WINDOW_MS } from "./assigned-work-invariant.js"; export { ASSIGNED_WORK_CONTINUED_ACTION, SCHEDULER_TRANSIENT_RETRY_REASON }; @@ -12,19 +12,23 @@ export async function taskAutomaticAttempts(db: Db, companyId: string, issueId: db.select({ createdAt: activityLog.createdAt, outcome: sql`${activityLog.details}->>'outcome'` }).from(activityLog).where(and( eq(activityLog.companyId, companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId), eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), gte(activityLog.createdAt, since), + sql`coalesce(${activityLog.details}->>'continuedFromSavedWork', '') <> 'true'`, sql`coalesce(${activityLog.details}->>${PAUSED_FOR_AUTOMATIC_WORK}, '') <> 'true'`)) .orderBy(desc(activityLog.createdAt)).limit(64), db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since))) + inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since), + startedOrPendingAutomaticRetry())) .orderBy(desc(heartbeatRuns.createdAt)).limit(64), ]); - const fresh = (await freshContinuationTimes(db, companyId, issueId, since)) - .filter(record => record.trigger !== "assigned_work").map(record => record.at); + const freshRecords = await freshContinuationTimes(db, companyId, issueId, since); + const sweepFresh = freshRecords.filter(record => record.trigger === "assigned_work").map(record => record.at); + const fresh = freshRecords.filter(record => record.trigger !== "assigned_work").map(record => record.at); + const sweep = [...continuations.map(row => row.createdAt), ...sweepFresh]; const others = [...transientRetries.map(row => row.createdAt), ...fresh]; - const all = [...continuations.map(row => row.createdAt), ...others]; - const started = [...continuations.filter(row => row.outcome !== "not_started").map(row => row.createdAt), ...others]; - return { continuations: continuations.map(row => row.createdAt), all, started }; + const all = [...sweep, ...others]; + const started = [...continuations.filter(row => row.outcome !== "not_started").map(row => row.createdAt), ...sweepFresh, ...others]; + return { continuations: sweep, all, started }; } export async function transientRetryWithinTaskCap(db: Db, companyId: string, issueId: string, at: Date) {