diff --git a/server/src/__tests__/failed-continuation-continues-again.test.ts b/server/src/__tests__/failed-continuation-continues-again.test.ts new file mode 100644 index 00000000000..ba7caebdcd2 --- /dev/null +++ b/server/src/__tests__/failed-continuation-continues-again.test.ts @@ -0,0 +1,417 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { activityLog, agentWakeupRequests, agents, companies, createDb, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { recoveryService } from "../services/recovery/service.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION, assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; +import { FRESH_CONTINUATION_SCHEMA, continueHeldExecutions, ensureFreshContinuation, resetBacklogContinuationPacingForTests } from "../services/fresh-continuation-fallback.js"; +import { PERMANENT_FAILURE_CODES, PERSON_OWNED_FAILURE_CODES } from "../services/continuation-cause.js"; +import { AUTOMATIC_RETRY_REASONS, SHARED_CONTINUATION_WINDOW_MS } from "../services/recovery/continuation-ledger.js"; +import { SERVER_RESTART_RETRY_MAX_ATTEMPTS, SERVER_RESTART_RETRY_REASON, SERVER_RESTART_RETRY_WINDOW_MS } from "../services/execution-recovery-attempt.js"; +import { FRESH_CONTINUATIONS_PER_HOUR } from "../services/fresh-continuation-fallback.js"; +import { createSystemRecovery } from "../services/system-recovery-state.js"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; + +const describePostgres = recoveryPostgresSupport.supported ? describe : describe.skip; +if (!recoveryPostgresSupport.supported) console.warn(`Skipping failed continuation tests: ${recoveryPostgresSupport.reason}`); + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const DAYTONA_TIMEOUT = "HTTP request timed out after 300000ms waiting for a response. This is a client-side deadline (DaytonaConfig.timeout)"; + +describe("shared continuation ledger", () => { + it("uses the server-restart lane's window, cap and retry reason", () => { + expect(SHARED_CONTINUATION_WINDOW_MS).toBe(SERVER_RESTART_RETRY_WINDOW_MS); + expect(FRESH_CONTINUATIONS_PER_HOUR).toBe(SERVER_RESTART_RETRY_MAX_ATTEMPTS); + expect(AUTOMATIC_RETRY_REASONS).toContain(SERVER_RESTART_RETRY_REASON); + }); +}); + +describePostgres("a continuation that fails before doing any work is continued again", () => { + let temporary: Awaited>; + let db: ReturnType; + + beforeAll(async () => { + temporary = await startRecoveryPostgres("failed-continuation-"); + db = createDb(temporary.connectionString); + }, 240_000); + + afterAll(async () => { + await db?.$client.end({ timeout: 0 }); + await temporary?.cleanup(); + }); + + function wakeQueue() { + const calls: string[] = []; + const enqueueWakeup = vi.fn(async (agentId: string, opts?: { payload?: Record | null; reason?: string | null }) => { + const issueId = String(opts?.payload?.issueId); + const [issue] = await db.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, issueId)); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: issue!.companyId, agentId, source: "automation", + triggerDetail: "system", reason: opts?.reason ?? null, payload: opts?.payload ?? null, status: "queued" }).returning(); + calls.push(issueId); + return { id: wake!.id } as never; + }); + return { calls, enqueueWakeup }; + } + + async function seed() { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Continuation fixture", issuePrefix: `C${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Builder", adapterType: "paperclip_runner", status: "idle", adapterConfig: {} }); + let number = 0; + async function task(values: Partial = {}) { + number += 1; + const [row] = await db.insert(issues).values({ companyId, title: `Task ${number}`, status: "in_progress", assigneeAgentId: agentId, + issueNumber: number, identifier: `C${companyId.slice(0, 7)}-${number}`, updatedAt: new Date(Date.now() - 2 * HOUR), + createdAt: new Date(Date.now() - 5 * HOUR), ...values }).returning(); + return row!; + } + async function run(issueId: string, values: Partial = {}) { + const finishedAt = values.finishedAt ?? new Date(Date.now() - 10 * MINUTE); + const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", status: "failed", + runtimeMode: "legacy", startedAt: new Date(finishedAt.getTime() - MINUTE), finishedAt, + createdAt: new Date(finishedAt.getTime() - 2 * MINUTE), livenessState: "failed", + runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, ...values, + contextSnapshot: { issueId, taskId: issueId, ...(values.contextSnapshot ?? {}) } }).returning(); + return row!; + } + async function continued(issueId: string, values: Partial = {}) { + const source = await run(issueId, { errorCode: "native_restart_recovery_blocked", error: "native_interruption_deadline_exceeded", + finishedAt: new Date(Date.now() - 70 * MINUTE) }); + const recordId = randomUUID(); + const continuation = await run(issueId, { errorCode: "setup_failed", error: DAYTONA_TIMEOUT, retryOfRunId: source.id, ...values, + contextSnapshot: { wakeReason: "issue_recovery_action_restored", previousRunId: source.id, forceFreshSession: true, + freshContinuationIsolation: recordId, ...(values.contextSnapshot ?? {}) } }); + await db.insert(issueRecoveryActions).values({ id: recordId, companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "resolved", outcome: "restored", ownerType: "system", returnOwnerAgentId: agentId, cause: "fresh_continuation", + fingerprint: `fresh-continuation:${source.id}`, nextAction: "Continuing.", resolutionNote: "continued_fresh_from_last_saved", + resolvedAt: new Date(Date.now() - 60 * MINUTE), attemptCount: 1, + evidence: { runId: source.id, continuationDelivery: "delivered", continuationRunId: continuation.id, + freshContinuation: { schema: FRESH_CONTINUATION_SCHEMA, sourceRunId: source.id, isolation: "remote_epoch", + recordedAt: new Date(Date.now() - 60 * MINUTE).toISOString() } } }); + return continuation; + } + return { companyId, agentId, task, run, continued }; + } + + async function heldRun(f: Awaited>, errorCode: string, contextSnapshot: Record, + endedAgo = 2 * HOUR) { + const task = await f.task({ status: "blocked", blockedTransitionAt: new Date(Date.now() - 30 * MINUTE), statusVersion: 4 }); + const failed = await f.run(task.id, { runtimeMode: "native", nativeIssueId: task.id, issueId: task.id, errorCode, error: errorCode, + finishedAt: new Date(Date.now() - endedAgo), contextSnapshot }); + const [hold] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: task.id, kind: "active_run_watchdog", + status: "resolved", outcome: "blocked", resolvedAt: new Date(Date.now() - endedAgo), ownerType: "board", returnOwnerAgentId: f.agentId, + cause: "native_continuation_requires_reconciliation", fingerprint: `native-continuation:${failed.id}`, nextAction: "Held.", + evidence: { runId: failed.id, originalFailureCode: errorCode, nativeFailureBlock: { runId: failed.id, statusVersion: 4 }, + automaticRecovery: { policy: "preserve_without_replay_v1", runId: failed.id, replay: "blocked", actionOutcome: "unknown", + recordedAt: new Date(Date.now() - endedAgo).toISOString() } } }).returning(); + return { task, failed, hold: hold! }; + } + const recoveryContext = { wakeReason: "transient_failure_retry", retryReason: "transient_failure", previousRunId: randomUUID() }; + const personRetryContext = { wakeReason: "task_retry", previousRunId: randomUUID(), explicitUserContinuation: { previousRunId: randomUUID() } }; + + const taskRow = async (id: string) => (await db.select().from(issues).where(eq(issues.id, id)))[0]!; + const activeStrandedHolds = async (issueId: string) => db.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.sourceIssueId, issueId), eq(issueRecoveryActions.kind, "stranded_assigned_issue"), + inArray(issueRecoveryActions.status, ["active", "escalated"]))); + const continuationsOf = async (runId: string) => db.select().from(issueRecoveryActions) + .where(sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,sourceRunId}' = ${runId}`); + + it("a recorded continuation whose run failed in setup is not escalated to the board, and the sweep continues it again from saved work", async () => { + const f = await seed(); + const task = await f.task(); + const failed = await f.continued(task.id); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + + await recovery.reconcileStrandedAssignedIssues({ companyId: f.companyId, issueId: task.id, skipRelatedRecovery: true }); + expect((await taskRow(task.id)).status).toBe("in_progress"); + expect(await activeStrandedHolds(task.id)).toHaveLength(0); + + const check = await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id }); + expect(check).toMatchObject({ outcome: "repaired", verdict: { kind: "violation" } }); + const [record] = await continuationsOf(failed.id); + expect(record?.evidence).toMatchObject({ continuationDelivery: "pending", freshContinuation: { sourceRunId: failed.id } }); + expect(queue.calls).toHaveLength(0); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id })) + .toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent" } }); + }, 60_000); + + it("a todo task whose assignment recovery run failed in setup is continued, not blocked", async () => { + const f = await seed(); + const task = await f.task({ status: "todo" }); + const earlier = await f.run(task.id, { errorCode: "controller_lease_lost", finishedAt: new Date(Date.now() - 80 * MINUTE) }); + const failed = await f.run(task.id, { errorCode: "setup_failed", error: DAYTONA_TIMEOUT, retryOfRunId: earlier.id, + contextSnapshot: { wakeReason: "issue_assignment_recovery", retryReason: "assignment_recovery" } }); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + + await recovery.reconcileStrandedAssignedIssues({ companyId: f.companyId, issueId: task.id, skipRelatedRecovery: true }); + expect((await taskRow(task.id)).status).toBe("todo"); + expect(await activeStrandedHolds(task.id)).toHaveLength(0); + + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id })).toMatchObject({ outcome: "repaired" }); + expect(await continuationsOf(failed.id)).toHaveLength(1); + expect(queue.calls).toHaveLength(0); + }, 60_000); + + it("a task an older release already escalated to the board for a failed continuation is continued on the first sweep after deploy", async () => { + const f = await seed(); + const blockedAt = new Date(Date.now() - 20 * MINUTE); + const task = await f.task({ status: "blocked", blockedTransitionAt: blockedAt }); + const failed = await f.continued(task.id, { errorCode: "setup_failed", + error: "remote_workspace_sandbox_disk_full: the sandbox disk is full; free space in the sandbox before saving the work", + finishedAt: new Date(blockedAt.getTime() - 30_000) }); + const [hold] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: task.id, kind: "stranded_assigned_issue", + status: "active", ownerType: "board", cause: "stranded_assigned_issue", + fingerprint: `source_scoped_recovery:${f.companyId}:${task.id}:stranded_assigned_issue`, + evidence: { previousStatus: "in_progress", latestRunId: failed.id, latestRunErrorCode: "setup_failed", + routingPolicy: "board_escalation_no_takeover_v1" }, nextAction: "Board operator: inspect the evidence.", attemptCount: 1 }).returning(); + await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "system", action: "issue.updated", + entityType: "issue", entityId: task.id, details: { status: "blocked", previousStatus: "in_progress", + source: "recovery.reconcile_stranded_assigned_issue" }, createdAt: blockedAt }); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup }); + + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id)).toContain(task.id); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id })) + .toMatchObject({ outcome: "repaired", verdict: { reason: "recovery_block" } }); + expect((await taskRow(task.id)).status).not.toBe("blocked"); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, hold!.id)))[0]) + .toMatchObject({ status: "resolved", ownerType: "system" }); + expect(await continuationsOf(failed.id)).toHaveLength(1); + expect(queue.calls).toHaveLength(0); + }, 60_000); + + it.each([ + ["setup_failed", "Database query failed (40P01)"], + ["adapter_failed", "native_remote_recovery_lease_mismatch: stage=runner_verification field=replacement_sandbox"], + ["runner_remote_provider_artifact_incompatible", "provider pack verification failed (exitCode=1; timedOut=false)"], + ])("the held backlog continues a recovery run that failed with %s, and still leaves an ordinary run's provider failure", async (errorCode, error) => { + resetBacklogContinuationPacingForTests(db); + const f = await seed(); + async function held(contextSnapshot: Record) { + const task = await f.task({ status: "blocked", blockedTransitionAt: new Date(Date.now() - 30 * MINUTE), statusVersion: 4 }); + const failed = await f.run(task.id, { runtimeMode: "native", nativeIssueId: task.id, issueId: task.id, errorCode, error, + finishedAt: new Date(Date.now() - 2 * HOUR), contextSnapshot }); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: task.id, kind: "active_run_watchdog", + status: "resolved", outcome: "blocked", resolvedAt: new Date(Date.now() - 2 * HOUR), ownerType: "board", returnOwnerAgentId: f.agentId, + cause: "native_continuation_requires_reconciliation", fingerprint: `native-continuation:${failed.id}`, nextAction: "Held.", + evidence: { runId: failed.id, originalFailureCode: errorCode, nativeFailureBlock: { runId: failed.id, statusVersion: 4 }, + automaticRecovery: { policy: "preserve_without_replay_v1", runId: failed.id, replay: "blocked", actionOutcome: "unknown", + recordedAt: new Date(Date.now() - 2 * HOUR).toISOString() } } }); + return failed; + } + const recoveryRun = await held({ wakeReason: "transient_failure_retry", retryReason: "transient_failure", previousRunId: randomUUID() }); + const ordinary = await held({ wakeReason: "issue_assigned" }); + for (let pass = 0; pass < 3; pass += 1) await continueHeldExecutions(db, new Date()); + expect(await continuationsOf(recoveryRun.id)).toHaveLength(1); + expect(await continuationsOf(ordinary.id)).toHaveLength(0); + }, 60_000); + + it("a continuation run that failed for a reason a person owns is still escalated as before", async () => { + const f = await seed(); + const task = await f.task(); + await f.continued(task.id, { errorCode: "provider_tool_definition_invalid", error: "The tool definition was rejected by the provider." }); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + await recovery.reconcileStrandedAssignedIssues({ companyId: f.companyId, issueId: task.id, skipRelatedRecovery: true }); + expect((await taskRow(task.id)).status).toBe("blocked"); + }, 60_000); + + it("past the hourly cap the task waits for the next window, unblocked, and no attempt is spent", async () => { + const f = await seed(); + const task = await f.task(); + const failed = await f.continued(task.id); + for (let index = 0; index < 6; index += 1) { + await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "system", action: ASSIGNED_WORK_CONTINUED_ACTION, + entityType: "issue", entityId: task.id, details: { outcome: "queued" }, createdAt: new Date(Date.now() - (50 - index * 5) * MINUTE) }); + } + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + await recovery.reconcileStrandedAssignedIssues({ companyId: f.companyId, issueId: task.id, skipRelatedRecovery: true }); + const check = await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id }); + expect(check).toMatchObject({ outcome: "deferred_cap" }); + expect(check!.retryAt!.getTime()).toBeGreaterThan(Date.now()); + expect((await taskRow(task.id)).status).toBe("in_progress"); + expect(await continuationsOf(failed.id)).toHaveLength(0); + expect(queue.calls).toHaveLength(0); + }, 60_000); + it("a recovery run stopped during preparation with no hold left is continued once the wait bound passes, not waited on forever", async () => { + const f = await seed(); + const task = await f.task(); + const endedAt = new Date(Date.now() - 5 * MINUTE); + const stopped = await f.run(task.id, { status: "interrupted", errorCode: "server_shutdown_interrupted", executionStage: "preparing", + error: "Interrupted by graceful server shutdown (SIGTERM)", finishedAt: endedAt, + contextSnapshot: { wakeReason: "issue_assignment_recovery", retryReason: "assignment_recovery" } }); + const queue = wakeQueue(); + let clock = new Date(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup, now: () => clock }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: task.id })).toMatchObject({ outcome: "waiting_on_recovery" }); + expect(await continuationsOf(stopped.id)).toHaveLength(0); + for (const step of [3, 6, 12]) { + clock = new Date(Date.now() + step * MINUTE); + await sweep.checkIssue({ companyId: f.companyId, issueId: task.id }); + } + expect(await continuationsOf(stopped.id)).toHaveLength(1); + expect(queue.calls).toHaveLength(0); + }, 60_000); + it.each([["20 minutes", 20 * MINUTE, "continued"], ["30 hours", 30 * HOUR, "retry_needed"]] as const)( + "a saved-work wait on a hold nobody records a check on ends in a named state; run ended %s ago, checked by two servers", async (_label, endedAgo, ending) => { + const f = await seed(); + const task = await f.task(); + const failed = await f.run(task.id, { errorCode: "setup_failed", error: DAYTONA_TIMEOUT, finishedAt: new Date(Date.now() - endedAgo), + contextSnapshot: { wakeReason: "transient_failure_retry", retryReason: "transient_failure" } }); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: task.id, kind: "active_run_watchdog", + status: "resolved", outcome: "restored", ownerType: "system", returnOwnerAgentId: f.agentId, + cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${failed.id}`, nextAction: "Recovering saved work.", + resolvedAt: new Date(Date.now() - endedAgo), evidence: { runId: failed.id, systemRecovery: createSystemRecovery(failed.id) } }); + const queue = wakeQueue(); + let clock = new Date(); + const servers = [0, 1].map(() => assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup, now: () => clock })); + for (const [index, step] of [0, 4, 9, 17, 33].entries()) { + clock = new Date(Date.now() + step * MINUTE); + await servers[index % 2]!.checkIssue({ companyId: f.companyId, issueId: task.id }); + if ((await continuationsOf(failed.id)).length || queue.calls.length) break; + } + if (ending === "continued") { + expect(await continuationsOf(failed.id)).toHaveLength(1); + expect(queue.calls).toHaveLength(0); + } else { + expect(await continuationsOf(failed.id)).toHaveLength(0); + const [entry] = await db.select().from(activityLog).where(and(eq(activityLog.entityId, task.id), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION))); + expect(entry?.details).toMatchObject({ outcome: "queued" }); + expect(queue.calls).toHaveLength(1); + } + }, 60_000); + it("the held backlog leaves each permanent cause blocked", async () => { + resetBacklogContinuationPacingForTests(db); + const f = await seed(); + const left = []; + for (const code of PERMANENT_FAILURE_CODES) left.push(await heldRun(f, code, recoveryContext)); + for (let pass = 0; pass < 4; pass += 1) await continueHeldExecutions(db, new Date()); + for (const entry of left) { + expect(await continuationsOf(entry.failed.id)).toHaveLength(0); + expect((await taskRow(entry.task.id)).status).toBe("blocked"); + } + }, 120_000); + + const runKinds = { + personRetry: personRetryContext, + automatic: recoveryContext, + firstAssignment: { wakeReason: "issue_assigned" }, + } as const; + const causes = { setup: "setup_failed", restart: "process_lost", permanent: "low_trust_boundary_mismatch", person: "codex_auth_required" } as const; + const decision = (kind: keyof typeof runKinds, cause: keyof typeof causes) => + cause === "restart" || (kind === "automatic" && cause === "setup"); + it.each(Object.keys(runKinds).flatMap(kind => Object.keys(causes).map(cause => [kind, cause] as [keyof typeof runKinds, keyof typeof causes])))( + "every entry point makes the same decision for a %s run that failed with the %s cause", async (kind, cause) => { + resetBacklogContinuationPacingForTests(db); + const f = await seed(); + const errorCode = causes[cause]; + const continues = decision(kind, cause); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + + const held = await heldRun(f, errorCode, runKinds[kind]); + await continueHeldExecutions(db, new Date()); + expect(await continuationsOf(held.failed.id)).toHaveLength(continues ? 1 : 0); + + const blockedAt = new Date(Date.now() - 20 * MINUTE); + const escalated = await f.task({ status: "blocked", blockedTransitionAt: blockedAt }); + const escalatedRun = await f.run(escalated.id, { errorCode, error: errorCode, finishedAt: new Date(blockedAt.getTime() - 30_000), + contextSnapshot: runKinds[kind] }); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: escalated.id, kind: "stranded_assigned_issue", + status: "active", ownerType: "board", cause: "stranded_assigned_issue", + fingerprint: `source_scoped_recovery:${f.companyId}:${escalated.id}:stranded_assigned_issue`, + evidence: { previousStatus: "in_progress", latestRunId: escalatedRun.id, latestRunErrorCode: errorCode }, nextAction: "Held.", attemptCount: 1 }); + await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "system", action: "issue.updated", + entityType: "issue", entityId: escalated.id, details: { status: "blocked", previousStatus: "in_progress", + source: "recovery.reconcile_stranded_assigned_issue" }, createdAt: blockedAt }); + const sweep = await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: escalated.id }); + expect(sweep?.outcome === "left_to_hold_owner").toBe(!continues); + const fresh = await continuationsOf(escalatedRun.id); + if (continues && kind === "automatic") { + expect(fresh).toHaveLength(1); + expect(queue.calls).not.toContain(escalated.id); + } + if (!continues) { + expect(fresh).toHaveLength(0); + expect(queue.calls).not.toContain(escalated.id); + expect((await taskRow(escalated.id)).status).toBe("blocked"); + } + + if (kind === "automatic" && cause !== "restart") { + const open = await f.task(); + const failed = await f.run(open.id, { errorCode, error: errorCode, contextSnapshot: runKinds[kind] }); + await recovery.reconcileStrandedAssignedIssues({ companyId: f.companyId, issueId: open.id, skipRelatedRecovery: true }); + if (continues) { + expect((await taskRow(open.id)).status).toBe("in_progress"); + await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: open.id }); + expect(await continuationsOf(failed.id)).toHaveLength(1); + } + } + }, 120_000); + + it("no entry point continues a person-owned or permanent cause", async () => { + resetBacklogContinuationPacingForTests(db); + const f = await seed(); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + for (const code of [...PERSON_OWNED_FAILURE_CODES, ...PERMANENT_FAILURE_CODES]) { + const held = await heldRun(f, code, recoveryContext); + expect(await ensureFreshContinuation(db, { companyId: f.companyId, runId: held.failed.id, trigger: "held_recovery" })) + .toMatchObject({ kind: "person_owned" }); + const task = await f.task(); + const failed = await f.continued(task.id, { errorCode: code, error: code }); + await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: task.id }); + expect(await continuationsOf(failed.id)).toHaveLength(0); + } + await continueHeldExecutions(db, new Date()); + expect(await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, f.companyId), + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,trigger}' in ('held_recovery', 'assigned_work')`))).toHaveLength(0); + }, 240_000); + + it("six automatic attempts of mixed kinds in the hour stop the held backlog until the next window", async () => { + resetBacklogContinuationPacingForTests(db); + const f = await seed(); + const held = await heldRun(f, "setup_failed", recoveryContext); + const at = (minutesAgo: number) => new Date(Date.now() - minutesAgo * MINUTE); + for (const minutesAgo of [50, 40]) await f.run(held.task.id, { errorCode: "process_lost", scheduledRetryReason: "transient_failure", + createdAt: at(minutesAgo), finishedAt: at(minutesAgo - 1) }); + for (const minutesAgo of [35, 30]) await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "system", + action: ASSIGNED_WORK_CONTINUED_ACTION, entityType: "issue", entityId: held.task.id, details: { outcome: "queued" }, createdAt: at(minutesAgo) }); + for (const minutesAgo of [25, 20]) await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: held.task.id, + kind: "active_run_watchdog", status: "resolved", outcome: "restored", ownerType: "system", cause: "fresh_continuation", + fingerprint: `fresh-continuation:${randomUUID()}`, nextAction: "Continuing.", resolvedAt: at(minutesAgo), + evidence: { runId: randomUUID(), continuationDelivery: "delivered", freshContinuation: { schema: FRESH_CONTINUATION_SCHEMA, + trigger: "held_recovery", recordedAt: at(minutesAgo).toISOString() } } }); + await db.update(heartbeatRuns).set({ createdAt: at(10), finishedAt: at(9) }).where(eq(heartbeatRuns.id, held.failed.id)); + expect(await continueHeldExecutions(db, new Date())).toMatchObject({ continued: 0 }); + const [checked] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, held.hold.id)); + const nextCheckAt = Number((checked!.evidence.freshContinuationCheck as { nextCheckAt?: number }).nextCheckAt); + expect(nextCheckAt).toBeGreaterThan(Date.now() + 9 * MINUTE); + expect(nextCheckAt).toBeLessThanOrEqual(Date.now() + 10 * MINUTE + 5_000); + expect(await continuationsOf(held.failed.id)).toHaveLength(0); + expect(await continueHeldExecutions(db, new Date(Date.now() + 11 * MINUTE))).toMatchObject({ continued: 1 }); + }, 60_000); + it("checks that only waited for exact recovery spend no attempt, so the fallback is reached at the bound", async () => { + const f = await seed(); + const task = await f.task(); + const stopped = await f.run(task.id, { status: "interrupted", errorCode: "server_shutdown_interrupted", executionStage: "preparing", + error: "Interrupted by graceful server shutdown (SIGTERM)", finishedAt: new Date(Date.now() - 2 * MINUTE), + contextSnapshot: { wakeReason: "issue_assignment_recovery", retryReason: "assignment_recovery" } }); + for (const minutesAgo of [40, 36, 32, 28, 24]) await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", + actorId: "system", action: ASSIGNED_WORK_CONTINUED_ACTION, entityType: "issue", entityId: task.id, + details: { outcome: "queued" }, createdAt: new Date(Date.now() - minutesAgo * MINUTE) }); + const queue = wakeQueue(); + let clock = new Date(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup, now: () => clock }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: task.id })).toMatchObject({ outcome: "waiting_on_recovery" }); + clock = new Date(Date.now() + 16 * MINUTE); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: task.id })).toMatchObject({ outcome: "repaired" }); + expect(await continuationsOf(stopped.id)).toHaveLength(1); + }, 60_000); +}); diff --git a/server/src/__tests__/interrupted-preparation-hold-settle.test.ts b/server/src/__tests__/interrupted-preparation-hold-settle.test.ts index b79ca3c3ff9..6e5f2bb76bc 100644 --- a/server/src/__tests__/interrupted-preparation-hold-settle.test.ts +++ b/server/src/__tests__/interrupted-preparation-hold-settle.test.ts @@ -1,6 +1,6 @@ import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { and, eq, inArray, ne } from "drizzle-orm"; +import { and, eq, inArray, ne, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { activityLog, @@ -428,10 +428,9 @@ suite("held interrupted preparations on the shared ownership path", () => { for (let round = 0; round < 40; round += 1) { await heartbeat.reconcileStrandedAssignedIssues({ companyId, issueId }); const [task] = await db.select().from(issues).where(eq(issues.id, issueId)); - if (task!.status === "blocked") break; expect(task!.status).toBe(status); const [next] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, latest)); - expect(next).toBeDefined(); + if (!next) break; expect(next!.contextSnapshot).toMatchObject({ retryReason: "interrupted_preparation", executionRetryAccounting: { interruptedPreparations: successors + 1 } }); successors += 1; @@ -445,17 +444,21 @@ suite("held interrupted preparations on the shared ownership path", () => { latest = next!.id; } expect(successors).toBe(cap); - expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ status: "blocked" }); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ status }); + await db.update(agentWakeupRequests).set({ status: "failed", finishedAt: new Date() }) + .where(and(eq(agentWakeupRequests.companyId, companyId), sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`)); + await db.update(heartbeatRuns).set({ scheduledRetryAt: new Date(Date.now() - 31 * 60_000) }) + .where(and(eq(heartbeatRuns.issueId, issueId), ne(heartbeatRuns.id, firstRunId))); + const check = await heartbeat.checkAssignedWorkInvariant({ companyId, issueId }); + expect(check).toMatchObject({ outcome: cap >= 6 ? "deferred_cap" : "deferred_rate_limit" }); + expect(check!.retryAt!.getTime()).toBeGreaterThan(Date.now()); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, latest))).toEqual([]); const receipt = await requestTaskRetry({ db, companyId, issueId, actorId: "local-board", actorSource: "local_implicit", idempotencyKey: randomUUID(), observedRunId: latest }); expect(receipt).toMatchObject({ accepted: true }); }, 120_000); - // The scheduler gives a settled lost preparation the interrupted_preparation - // lane; the synchronous budget check reads the server-restart bound of the - // replaced owner. When only the preparation bound is spent, the sweep must - // still report the exhaustion instead of leaving the task in_progress. - it("blocks the task when a replaced preparation's lane is spent even though the restart bound is not", async () => { + it("leaves the task to the assigned-work sweep, not blocked, when a replaced preparation's lane is spent", async () => { const heartbeat = heartbeatService(db); const controller = await deadController(); const issueId = randomUUID(), runId = randomUUID(); @@ -478,7 +481,9 @@ suite("held interrupted preparations on the shared ownership path", () => { await heartbeat.reconcileStrandedAssignedIssues({ companyId, issueId }); expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toEqual([]); - expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ status: "blocked" }); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ status: "todo" }); + expect(await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.sourceIssueId, issueId), + inArray(issueRecoveryActions.status, ["active", "escalated"])))).toEqual([]); }); it.each([ diff --git a/server/src/services/continuation-cause.ts b/server/src/services/continuation-cause.ts new file mode 100644 index 00000000000..f73093de6f8 --- /dev/null +++ b/server/src/services/continuation-cause.ts @@ -0,0 +1,140 @@ +import { sql, type SQL } from "drizzle-orm"; +import type { heartbeatRuns } from "@paperclipai/db"; + +/** + * Run failure codes that only a person can resolve (the test for each: did a + * person fail to do something, or did the system fail to carry it forward? Only + * the first belongs here; e.g. a lost carry-over of a Retry's authorization, + * `continuation_user_authorization_missing`, is the system's): they stopped the work, a + * credential or access they must share, a configuration choice they must make, + * a budget stop, or a provider usage limit that is waiting as designed. Every + * other code, including codes that do not exist yet, gets the fallback. + */ +export const PERSON_OWNED_FAILURE_CODES = [ + // A person stopped or paused the work. + "cancelled", "operator_cancelled", "operator_interrupted", "issue_cancelled", "issue_paused", "agent_paused", + // A credential, secret or access the person must share. + "configuration_incomplete", "slack_configuration_incomplete", "signing_secret_unconfigured", + "user_secret_missing", "user_secret_definition_missing", "user_secret_definition_inactive", + "github_access_denied", "github_repo_not_accessible", "github_auth_required", "project_access_denied", + "permission_denied", "responsible_user_unauthorized", "oauth_access_projection_disabled", + "acpx_auth_required", "codex_auth_required", "claude_auth_required", "provider_auth_failed", "hermes_gateway_auth_failed", + // A configuration choice the person must change. + "native_provider_model_rejected", "provider_tool_definition_invalid", "turn_limit_exhausted", + // Budget stops. + "budget_blocked", "budget_hard_stop", "budget_exceeded", + // Provider usage limits that wait (and resume) as designed. + "provider_quota", "provider_limit_wait", "native_provider_usage_limit", + // An approval the person must give. + "approval_required", "formal_approval_unavailable", +] as const; + +export function isPersonOwnedFailureCode(code: string | null | undefined): boolean { + return typeof code === "string" && (PERSON_OWNED_FAILURE_CODES as readonly string[]).includes(code); +} + +export const AUTOMATIC_CONTINUATION_WAKE_REASONS = [ + "issue_continuation_needed", "issue_assignment_recovery", "issue_recovery_action_restored", "transient_failure_retry", +] as const; + +export const PERMANENT_FAILURE_CODES = [ + "agent_interaction_delivery_denied", "adapter_engine_unavailable", "agent_not_invokable", "agent_not_found", + "low_trust_isolation_unavailable", "low_trust_requires_isolated_workspace", "low_trust_boundary_mismatch", + "low_trust_requires_sandbox_environment", "low_trust_runtime_services_denied", +] as const; + +export const RESTART_CLASS_FAILURE_CODES = [ + "native_restart_recovery_blocked", "native_continuation_requires_reconciliation", "native_session_interrupted", + "native_runner_process_exited", "server_shutdown_interrupted", "process_lost", "controller_lease_lost", + "execution_finalization_deadline_exceeded", "local_runtime_api_listener_unavailable", + "runner_remote_process_identity_unavailable", "continuation_user_authorization_missing", +] as const; + +const REUSABLE_SANDBOX_RESUME_FAILED = "reusable_sandbox_resume_failed"; +const TRANSIENT_SANDBOX_CAUSES = ["transport_timeout", "transport_closed", "sandbox_state_transition", "provider_rate_limited", + "provider_unavailable"] as const; +const EXTERNAL_STOP_UNCONFIRMED_PROVIDERS = ["openai_dot", "muse"] as const; +const UNFINISHED_RUN_STATUSES = ["failed", "timed_out", "interrupted", "cancelled"]; + +type ClassifiedRun = Pick & { error?: string | null; runnerProfileJson?: unknown; resultJson?: unknown }; + +const TRANSIENT_SETUP_FAILURE_CODES = [ + REUSABLE_SANDBOX_RESUME_FAILED, "remote_workspace_prepare_timeout", "environment_provider_unavailable", + "workspace_git_scan_timeout", "workspace_git_scan_saturated", "repository_catalog_unavailable", +] as const; + +const object = (value: unknown): Record => + value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + +export function isSystemOwnedFailureCode(code: string | null | undefined): boolean { + return typeof code === "string" && code.length > 0 && !isPersonOwnedFailureCode(code) + && !(PERMANENT_FAILURE_CODES as readonly string[]).includes(code); +} + +export function continuationCauseIsSystemOwned(run: Pick): boolean { + if (!UNFINISHED_RUN_STATUSES.includes(run.status) || !isSystemOwnedFailureCode(run.errorCode)) return false; + const provider = object(object(object(run.runnerProfileJson).nativeExecutionInput).provider).kind; + if ((EXTERNAL_STOP_UNCONFIRMED_PROVIDERS as readonly unknown[]).includes(provider)) return false; + if (run.errorCode !== REUSABLE_SANDBOX_RESUME_FAILED) return true; + const cause = /\(cause=([a-z_]+)/.exec(run.error ?? "")?.[1]; + return cause !== undefined && (TRANSIENT_SANDBOX_CAUSES as readonly string[]).includes(cause); +} + +export function isRecoveryRun(run: Pick): boolean { + const context = object(run.contextSnapshot); + if (context.explicitUserContinuation || context.wakeReason === "task_retry") return false; + return Boolean(run.retryOfRunId || run.scheduledRetryReason || context.previousRunId || context.retryReason + || context.freshContinuationIsolation + || (AUTOMATIC_CONTINUATION_WAKE_REASONS as readonly unknown[]).includes(context.wakeReason)); +} + +export function canContinueRecoveryRunAutomatically(run: ClassifiedRun): boolean { + return continuationCauseIsSystemOwned(run) && isRecoveryRun(run); +} + +function firstRunFailureContinues(run: Pick): boolean { + if (isRestartClassFailureCode(run.errorCode)) return true; + const recovery = object(object(run.resultJson).executionRecovery); + return (TRANSIENT_SETUP_FAILURE_CODES as readonly unknown[]).includes(run.errorCode) + && recovery.kind === "bootstrap" && recovery.providerWorkStarted === false; +} + +export function continuesAutomatically(run: ClassifiedRun): boolean { + return continuationCauseIsSystemOwned(run) && (isRecoveryRun(run) || firstRunFailureContinues(run)); +} + +export function isRestartClassFailureCode(code: string | null | undefined): boolean { + return typeof code === "string" && (RESTART_CLASS_FAILURE_CODES as readonly string[]).includes(code); +} + +const list = (values: readonly string[]) => sql.join(values.map(value => sql`${value}`), sql`, `); + +export function continuationCauseIsSystemOwnedSql(alias: string): SQL { + const column = (name: string) => sql.raw(`${alias}.${name}`); + const code = column("error_code"); + return sql`(${column("status")} in (${list(UNFINISHED_RUN_STATUSES)}) and ${code} is not null and ${code} <> '' + and ${code} not in (${list([...PERSON_OWNED_FAILURE_CODES, ...PERMANENT_FAILURE_CODES])}) + and coalesce(${column("runner_profile_json")}#>>'{nativeExecutionInput,provider,kind}', '') not in (${list(EXTERNAL_STOP_UNCONFIRMED_PROVIDERS)}) + and (${code} <> ${REUSABLE_SANDBOX_RESUME_FAILED} + or substring(coalesce(${column("error")}, '') from '\\(cause=([a-z_]+)') in (${list(TRANSIENT_SANDBOX_CAUSES)})))`; +} + +export function recoveryRunSql(alias: string): SQL { + const column = (name: string) => sql.raw(`${alias}.${name}`); + const context = column("context_snapshot"); + return sql`(not coalesce(${context} ? 'explicitUserContinuation', false) + and coalesce(${context}->>'wakeReason', '') <> 'task_retry' + and (${column("retry_of_run_id")} is not null or ${column("scheduled_retry_reason")} is not null + or ${context} ? 'previousRunId' or ${context} ? 'retryReason' or ${context} ? 'freshContinuationIsolation' + or ${context}->>'wakeReason' in (${list(AUTOMATIC_CONTINUATION_WAKE_REASONS)})))`; +} + +export function continuesAutomaticallySql(alias: string): SQL { + const column = (name: string) => sql.raw(`${alias}.${name}`); + return sql`(${continuationCauseIsSystemOwnedSql(alias)} and (${recoveryRunSql(alias)} + or ${column("error_code")} in (${list(RESTART_CLASS_FAILURE_CODES)}) + or (${column("error_code")} in (${list(TRANSIENT_SETUP_FAILURE_CODES)}) + and ${column("result_json")}#>>'{executionRecovery,kind}' = 'bootstrap' + and ${column("result_json")}#>>'{executionRecovery,providerWorkStarted}' = 'false')))`; +} diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 2c5bcac3257..39cb5d8d485 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -26,6 +26,10 @@ import { notifyRoutineExecutionStatus } from "./routine-execution-status-notific import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { readRunCancellation } from "./run-cancellation.js"; import { readSystemRecovery } from "./system-recovery-state.js"; +import { RESTART_CLASS_FAILURE_CODES, continuationCauseIsSystemOwned, continuesAutomaticallySql, isPersonOwnedFailureCode } from "./continuation-cause.js"; +import { sharedContinuationAttempts } from "./recovery/continuation-ledger.js"; + +export { AUTOMATIC_CONTINUATION_WAKE_REASONS, PERSON_OWNED_FAILURE_CODES, isPersonOwnedFailureCode, isRecoveryRun } from "./continuation-cause.js"; /** * Fresh-continuation fallback: exact resumption of an unfinished run is an @@ -80,38 +84,6 @@ export const FRESH_CONTINUATION_SAVED_WORK_BOUND_MS = 30 * 60_000; export const NATIVE_INTERRUPTION_NO_PROGRESS_DEADLINE_MS = FRESH_CONTINUATION_PROGRESS_BOUND_MS; const EXACT_RECOVERY_RECHECK_MS = 60_000; -/** - * Run failure codes that only a person can resolve (the test for each: did a - * person fail to do something, or did the system fail to carry it forward? Only - * the first belongs here; e.g. a lost carry-over of a Retry's authorization, - * `continuation_user_authorization_missing`, is the system's): they stopped the work, a - * credential or access they must share, a configuration choice they must make, - * a budget stop, or a provider usage limit that is waiting as designed. Every - * other code, including codes that do not exist yet, gets the fallback. - */ -export const PERSON_OWNED_FAILURE_CODES = [ - // A person stopped or paused the work. - "cancelled", "operator_cancelled", "operator_interrupted", "issue_cancelled", "issue_paused", "agent_paused", - // A credential, secret or access the person must share. - "configuration_incomplete", "slack_configuration_incomplete", "signing_secret_unconfigured", - "user_secret_missing", "user_secret_definition_missing", "user_secret_definition_inactive", - "github_access_denied", "github_repo_not_accessible", "github_auth_required", "project_access_denied", - "permission_denied", "responsible_user_unauthorized", "oauth_access_projection_disabled", - "acpx_auth_required", "codex_auth_required", "claude_auth_required", "provider_auth_failed", "hermes_gateway_auth_failed", - // A configuration choice the person must change. - "native_provider_model_rejected", "provider_tool_definition_invalid", "turn_limit_exhausted", - // Budget stops. - "budget_blocked", "budget_hard_stop", "budget_exceeded", - // Provider usage limits that wait (and resume) as designed. - "provider_quota", "provider_limit_wait", "native_provider_usage_limit", - // An approval the person must give. - "approval_required", "formal_approval_unavailable", -] as const; - -export function isPersonOwnedFailureCode(code: string | null | undefined): boolean { - return typeof code === "string" && (PERSON_OWNED_FAILURE_CODES as readonly string[]).includes(code); -} - const UNFINISHED_STATUSES = ["failed", "timed_out", "interrupted", "cancelled"]; const LIVE_RUN_STATUSES = ["queued", "running", "scheduled_retry"]; const EXACT_RECOVERY_STATES = ["scheduled", "running", "awaiting_evidence"]; @@ -129,7 +101,7 @@ const EXACT_RECOVERY_STATES = ["scheduled", "running", "awaiting_evidence"]; */ export type FreshContinuationIsolation = "remote_epoch" | "writer_ended" | "fresh_isolated_workspace"; -export type FreshContinuationTrigger = "recovery_refused" | "held_recovery" | "continuation_wake"; +export type FreshContinuationTrigger = "recovery_refused" | "held_recovery" | "continuation_wake" | "assigned_work"; export type FreshContinuationResult = | { kind: "continued"; recoveryActionId: string; delivery: "recorded" | "caller_owns" | "saved_input"; isolation: FreshContinuationIsolation } @@ -473,18 +445,25 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< ...holds.filter(hold => readSystemRecovery(hold.evidence)).map(hold => hold.updatedAt.getTime())); const savedWorkSignature = savedWork.map(recovery => `${recovery!.generationId}:${recovery!.revision}:${recovery!.attempts}:${recovery!.state}`) .sort().join("|"); - const waitedMs = pendingMarker ? now.getTime() - Number(pendingMarker.firstPendingAt) : 0; + const sourceAgeMs = now.getTime() - run.finishedAt.getTime(); + const waitedMs = pendingMarker ? now.getTime() - Number(pendingMarker.firstPendingAt) + : sourceAgeMs <= BACKLOG_CONTINUATION_MAX_AGE_MS ? sourceAgeMs : 0; const progressing = Boolean(pendingMarker && ((typeof pendingMarker.progressAt === "number" && progressAt > pendingMarker.progressAt) || (typeof pendingMarker.savedWorkSignature === "string" && savedWorkSignature !== pendingMarker.savedWorkSignature))); // Saved-work recovery that is running and advancing keeps its own 30-minute // budget: it may still rescue unsaved files. Anything idle gets the normal bound. const savedWorkAdvancing = progressing && savedWork.some(recovery => recovery!.state === "running"); - const waitBounded = savedWorkAdvancing ? waitedMs >= FRESH_CONTINUATION_SAVED_WORK_BOUND_MS - : waitedMs >= FRESH_CONTINUATION_PROGRESS_BOUND_MS || (waitedMs >= FRESH_CONTINUATION_PENDING_BOUND_MS && !progressing) + const boundedAfter = (waited: number) => savedWorkAdvancing ? waited >= FRESH_CONTINUATION_SAVED_WORK_BOUND_MS + : waited >= FRESH_CONTINUATION_PROGRESS_BOUND_MS || (waited >= FRESH_CONTINUATION_PENDING_BOUND_MS && !progressing) || (personAsked && !savedWork.some(recovery => EXACT_RECOVERY_STATES.includes(recovery!.state))); + const waitBounded = boundedAfter(waitedMs); + const markerBounded = boundedAfter(pendingMarker ? waitedMs : 0); // Rechecks back off 1, 2, 3, 4 minutes, so the bound is reached on time. const step = Math.min(4, (typeof pendingMarker?.checks === "number" ? pendingMarker.checks : 0) + 1) * EXACT_RECOVERY_RECHECK_MS; - const pending = (reason: string, retryAt: Date): FreshContinuationResult | null => waitBounded ? null + let waitSkipped = false; + const pastBound = () => markerBounded || waitSkipped; + const pending = (reason: string, retryAt: Date): FreshContinuationResult | null => waitBounded ? (waitSkipped = true, null) + : !pendingMarker && sourceAgeMs > BACKLOG_CONTINUATION_MAX_AGE_MS ? { kind: "person_owned", reason: "retry_needed" } : { kind: "exact_recovery_pending", reason, progressAt, savedWorkSignature, retryAt: new Date(Math.min(retryAt.getTime(), now.getTime() + step)) }; @@ -532,11 +511,12 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< if (firstWait) return firstWait; // Past the bound, a hold that workspace recovery still owns is settled too: // the continuation never uses that workspace (isolation below). - const owned = waitBounded ? holds.filter(hold => isOwnedReconciliationHold(hold) + const owned = pastBound() ? holds.filter(hold => isOwnedReconciliationHold(hold) || (isRestoreHold(hold) && blocksExecution(hold) && !isHumanGate(hold))) : ownedHolds; // (b) Causes that are the person's. - if (isPersonOwnedFailureCode(run.errorCode)) return { kind: "person_owned", reason: run.errorCode! }; + if (isPersonOwnedFailureCode(run.errorCode) || (run.errorCode && !continuationCauseIsSystemOwned(run))) + return { kind: "person_owned", reason: run.errorCode! }; // A Stop a person (or an agent acting on the task) asked for is never undone. const cancellation = readRunCancellation(run.resultJson); if ((cancellation && ["user", "agent"].includes(cancellation.initiator.type)) @@ -571,16 +551,9 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< } // (c) Bounded per task: the restart lane's hourly cap, then back off. - const windowStart = new Date(now.getTime() - FRESH_CONTINUATION_WINDOW_MS).toISOString(); - const recent = await tx.select({ recordedAt: sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}'` }) - .from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, companyId), - eq(issueRecoveryActions.sourceIssueId, issue.id), - sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,schema}' = ${FRESH_CONTINUATION_SCHEMA}`, - sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}' > ${windowStart}`)) - .orderBy(asc(sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}'`)); - if (recent.length >= FRESH_CONTINUATIONS_PER_HOUR) { - const oldest = new Date(recent[0]!.recordedAt).getTime(); - return { kind: "backoff", retryAt: new Date(oldest + FRESH_CONTINUATION_WINDOW_MS) }; + const attempts = await sharedContinuationAttempts(tx, companyId, issue.id, now); + if (attempts.length >= FRESH_CONTINUATIONS_PER_HOUR) { + return { kind: "backoff", retryAt: new Date(attempts[0]!.getTime() + FRESH_CONTINUATION_WINDOW_MS) }; } // Saved user input (a message or a Retry the person pressed) continues the @@ -615,7 +588,7 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< const fenced = await fenceSourceWorkspace(tx, run); // Never claim a stop that was not proven: record what isolates the successor. const isolation: FreshContinuationIsolation = fenced || remoteSource ? "remote_epoch" - : writer === "ended" && !waitBounded && !refused ? "writer_ended" : "fresh_isolated_workspace"; + : writer === "ended" && !pastBound() && !refused ? "writer_ended" : "fresh_isolated_workspace"; if (isolation === "fresh_isolated_workspace" && delivery === "saved_input") delivery = "recorded"; const unverified = await unverifiedActions(tx, run); const created = await createdObjects(tx, run); @@ -814,12 +787,7 @@ export function heldExecutionsAwaitingFreshContinuation(db: Db, now: Date, limit * through restart recovery and the disposition pass, not through this list. */ export const BACKLOG_CONTINUATION_MAX_AGE_MS = 24 * 60 * 60_000; -export const BACKLOG_CONTINUATION_SOURCE_CODES = [ - "native_restart_recovery_blocked", "native_continuation_requires_reconciliation", "native_session_interrupted", - "native_runner_process_exited", "server_shutdown_interrupted", "process_lost", "controller_lease_lost", - "execution_finalization_deadline_exceeded", "local_runtime_api_listener_unavailable", - "runner_remote_process_identity_unavailable", "continuation_user_authorization_missing", -] as const; +export const BACKLOG_CONTINUATION_SOURCE_CODES = RESTART_CLASS_FAILURE_CODES; /** At most this many backlog continuations per server per rolling minute, and per pass. */ export const BACKLOG_CONTINUATIONS_PER_MINUTE = 10; const BACKLOG_CONTINUATIONS_PER_PASS = 5; @@ -845,7 +813,7 @@ function interruptionEligibility(input: { now: Date; companyId: SQL; issueId: SQ and retry.run_id is null and retry.payload->>'issueId' = ${input.issueId}::text and retry.requested_at > source.finished_at) or exists (select 1 from ${heartbeatRuns} held where ${refusedLater} and held.context_snapshot->>'wakeReason' = 'task_retry' and held.finished_at > ${since}))`); - const recent = source(sql`source.error_code in (${sql.join(BACKLOG_CONTINUATION_SOURCE_CODES.map(code => sql`${code}`), sql`, `)}) + const recent = source(sql`${continuesAutomaticallySql("source")} and greatest(source.finished_at, (select max(held.finished_at) from ${heartbeatRuns} held where ${refusedLater})) > ${since}`); return { recent, personAsked }; } diff --git a/server/src/services/recovery/assigned-work-invariant.ts b/server/src/services/recovery/assigned-work-invariant.ts index c94361b777f..518287850bd 100644 --- a/server/src/services/recovery/assigned-work-invariant.ts +++ b/server/src/services/recovery/assigned-work-invariant.ts @@ -355,17 +355,19 @@ function waitingReason(facts: AssignedWorkFacts, now: Date): WaitingReason | nul * rolling hour (the server-restart lane's bound). Below the cap the spacing * doubles with each continuation in the hour, 2, 4, 8 and then 15 minutes at * most. At the cap the next one waits until the oldest leaves the hour. */ -export function continuationAllowance(recent: Date[], now: Date = new Date()): +export function continuationAllowance(recent: Date[], now: Date = new Date(), spacedBy: Date[] = recent): { allowed: true } | { allowed: false; retryAt: Date; capped: boolean } { - const inWindow = recent.filter(at => at.getTime() > now.getTime() - CONTINUATION_CAP_WINDOW_MS) + const windowed = (dates: Date[]) => dates.filter(at => at.getTime() > now.getTime() - CONTINUATION_CAP_WINDOW_MS) .sort((a, b) => b.getTime() - a.getTime()); + const inWindow = windowed(recent); if (inWindow.length >= CONTINUATION_CAP) { const oldest = inWindow[CONTINUATION_CAP - 1]; return { allowed: false, capped: true, retryAt: new Date(oldest.getTime() + CONTINUATION_CAP_WINDOW_MS) }; } - if (inWindow.length > 0) { - const spacing = Math.min(CONTINUATION_MAX_BACKOFF_MS, CONTINUATION_BASE_BACKOFF_MS * 2 ** (inWindow.length - 1)); - const next = inWindow[0].getTime() + spacing; + const spacingWindow = windowed(spacedBy); + if (spacingWindow.length > 0) { + const spacing = Math.min(CONTINUATION_MAX_BACKOFF_MS, CONTINUATION_BASE_BACKOFF_MS * 2 ** (spacingWindow.length - 1)); + const next = spacingWindow[0].getTime() + spacing; if (next > now.getTime()) return { allowed: false, capped: false, retryAt: new Date(next) }; } return { allowed: true }; diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 03f8843f32b..9128d888876 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -33,6 +33,8 @@ import { workspaceRecoverySourceEligible } from "../workspace-recovery-source.js import { PENDING_CONTINUATION_BOUND_MS } from "../recovery-continuation-pacing.js"; import { attributeBlock, BLOCK_ATTRIBUTION_WINDOW_MS, blockedStatusDetailsSql } from "../interruption-block-release.js"; import { ASSIGNED_WORK_CONTINUED_ACTION, taskAutomaticAttempts } from "./task-automatic-attempts.js"; +import { ensureFreshContinuation, type FreshContinuationResult } from "../fresh-continuation-fallback.js"; +import { canContinueRecoveryRunAutomatically } from "../continuation-cause.js"; import { STRANDED_CONTINUATION_MAX_AGE_MS, STRANDED_KIND, strandedHoldContinuesAutomatically, type StrandedHoldRun } from "./stranded-hold-continuation.js"; import { continuationActivityMessage, @@ -99,7 +101,7 @@ export type AssignedWorkDeps = { export type AssignedWorkCheck = { verdict: AssignedWorkVerdict; outcome: "ok" | "settling" | "repaired" | "settled_lost_run" | "deferred_rate_limit" | "deferred_cap" | "stale_reported" | "left_to_hold_owner" | "left_to_native_recovery" | "changed" | "enqueue_refused" - | "started_saved_work_recovery" | "saved_work_recovery_busy"; + | "waiting_on_recovery" | "started_saved_work_recovery" | "saved_work_recovery_busy"; retryAt: Date | null; }; @@ -304,7 +306,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { .map(action => action.evidence?.latestRunId).filter((id): id is string => typeof id === "string" && /^[0-9a-f-]{36}$/i.test(id)); const strandedRuns = new Map((strandedRunIds.length ? await database.select({ id: heartbeatRuns.id, agentId: heartbeatRuns.agentId, status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode, error: heartbeatRuns.error, - finishedAt: heartbeatRuns.finishedAt, + finishedAt: heartbeatRuns.finishedAt, retryOfRunId: heartbeatRuns.retryOfRunId, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, + contextSnapshot: heartbeatRuns.contextSnapshot, runnerProfileJson: heartbeatRuns.runnerProfileJson, executionRecovery: sql`${heartbeatRuns.resultJson}->'executionRecovery'` }) .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, strandedRunIds))) : []) .map(({ id, ...run }) => [id, run])); @@ -588,6 +591,16 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { await settleContinuationActivity(claimed, claimed.reason, "restored_to_review"); return { verdict: claimed.verdict, outcome: "repaired", retryAt: null }; } + const fresh = await continueFromSavedWork(issue, facts, at); + if (fresh?.kind === "continued") { + await settleContinuationActivity(claimed, claimed.reason, "queued", { continuedFromSavedWork: "true" }); + return { verdict: claimed.verdict, outcome: "repaired", retryAt: claimed.nextCheckAt }; + } + if (fresh?.kind === "backoff" || fresh?.kind === "exact_recovery_pending") { + if (claimed.restoredTo && fresh.kind === "exact_recovery_pending") await revertRestoredBlock(issue, claimed); + await settleContinuationActivity(claimed, claimed.reason, "not_started", { waitingUntil: fresh.retryAt.toISOString() }); + return { verdict: claimed.verdict, outcome: fresh.kind === "backoff" ? "deferred_cap" : "waiting_on_recovery", retryAt: fresh.retryAt }; + } const run = await deps.enqueueWakeup(issue.assigneeAgentId!, { source: facts.latestRun ? "automation" : "assignment", triggerDetail: "system", reason: facts.latestRun ? "issue_continuation_needed" : "issue_assigned", @@ -604,10 +617,27 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { // A block lifted for it is put back, so the task never sits unblocked // with nothing pending. if (!run && claimed.restoredTo) await revertRestoredBlock(issue, claimed); - await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started"); + await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started", + fresh?.kind === "person_owned" ? { waitingFor: fresh.reason } : {}); return { verdict: claimed.verdict, outcome: run ? "repaired" : "enqueue_refused", retryAt: claimed.nextCheckAt }; } + async function continueFromSavedWork(issue: IssueRow, facts: AssignedWorkFacts, at: Date): Promise { + const source = facts.latestRun; + if (!source?.finishedAt || !["failed", "timed_out", "interrupted", "cancelled"].includes(source.status) + || (source.status === "cancelled" && source.errorCode === "execution_reconciliation_required")) return null; + const [run] = await db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode, error: heartbeatRuns.error, + retryOfRunId: heartbeatRuns.retryOfRunId, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, + contextSnapshot: heartbeatRuns.contextSnapshot, runnerProfileJson: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.id, source.id))); + if (!run || !canContinueRecoveryRunAutomatically(run)) return null; + return ensureFreshContinuation(db, { companyId: issue.companyId, runId: source.id, trigger: "assigned_work", now: at }) + .catch(error => { + logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id }, "assigned-work continuation from saved work deferred"); + return null; + }); + } + async function startSavedWorkRecovery(issue: IssueRow, facts: AssignedWorkFacts, verdict: Extract, at: Date): Promise { const recovery = facts.savedWorkRecovery!; @@ -656,7 +686,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { || (current.status === "blocked" && fresh.activeRecoveryAction && !fresh.activeRecoveryAction.continuedBySweep)) return { kind: "changed" as const }; const attempts = await taskAutomaticAttempts(tx as unknown as Db, issue.companyId, issue.id, at); const recent = attempts.continuations; - const allowance = continuationAllowance(attempts.all, at); + const allowance = continuationAllowance(attempts.started, at, attempts.all); if (!allowance.allowed) return { kind: allowance.capped ? "capped" as const : "spaced" as const, retryAt: allowance.retryAt }; let restoredTo: Claimed["restoredTo"] = null; let restoredStatusVersion: number | null = null; @@ -687,7 +717,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { ...(restoredTo ? { status: restoredTo, previousStatus: "blocked" } : {}), attempt: recent.length + 1 }, }, publications); - const next = continuationAllowance([at, ...attempts.all], at); + const next = continuationAllowance([at, ...attempts.started], at, [at, ...attempts.all]); return { kind: "claimed" as const, activityId: activity.id, publications, verdict, reason: verdict.reason, restoredTo, restoredStatusVersion, blockedTransitionAt: current.blockedTransitionAt, unblockDescriptor: current.unblockDescriptor, blockedOwnerNotifiedAt: current.blockedOwnerNotifiedAt, @@ -745,10 +775,10 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { * admitted or refused; it is published then, never before. A refused * continuation stays a violation and is tried again under the same cap. */ async function settleContinuationActivity(claimed: { activityId: string; publications: ActivityPublication[] }, - reason: ViolationReason, outcome: "queued" | "not_started" | "restored_to_review") { - const update = outcome === "queued" ? { outcome, message: continuationActivityMessage(reason) } + reason: ViolationReason, outcome: "queued" | "not_started" | "restored_to_review", extra: Record = {}) { + const update = outcome === "queued" ? { outcome, message: continuationActivityMessage(reason), ...extra } : outcome === "restored_to_review" ? { outcome, message: CONTINUATION_REVIEW_MESSAGE } - : { outcome, message: CONTINUATION_REFUSED_MESSAGE }; + : { outcome, message: CONTINUATION_REFUSED_MESSAGE, ...extra }; await db.update(activityLog).set({ details: sql`coalesce(${activityLog.details}, '{}'::jsonb) || ${JSON.stringify(update)}::jsonb` }) .where(eq(activityLog.id, claimed.activityId)); for (const publication of claimed.publications) { diff --git a/server/src/services/recovery/continuation-ledger.ts b/server/src/services/recovery/continuation-ledger.ts new file mode 100644 index 00000000000..995f1eed742 --- /dev/null +++ b/server/src/services/recovery/continuation-ledger.ts @@ -0,0 +1,33 @@ +import { and, eq, gte, inArray, sql } from "drizzle-orm"; +import { activityLog, heartbeatRuns, issueRecoveryActions, type Db } from "@paperclipai/db"; + +export const ASSIGNED_WORK_CONTINUED_ACTION = "issue.continued_automatically"; +export const SCHEDULER_TRANSIENT_RETRY_REASON = "transient_failure"; +export const AUTOMATIC_RETRY_REASONS = [SCHEDULER_TRANSIENT_RETRY_REASON, "server_restart", "interrupted_preparation"]; +export const SHARED_CONTINUATION_WINDOW_MS = 60 * 60_000; + +export async function freshContinuationTimes(db: Db, companyId: string, issueId: string, since: Date) { + const rows = await db.select({ recordedAt: sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}'`, + trigger: sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,trigger}'` }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId), + sql`${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}' > ${since.toISOString()}`)).limit(64); + return rows.map(row => ({ at: new Date(row.recordedAt), trigger: row.trigger })) + .filter(record => Number.isFinite(record.at.getTime()) && record.at.getTime() > since.getTime()); +} + +export async function sharedContinuationAttempts(db: Db, companyId: string, issueId: string, at: Date): Promise { + const since = new Date(at.getTime() - SHARED_CONTINUATION_WINDOW_MS); + const [plainContinuations, retries, fresh] = await Promise.all([ + db.select({ createdAt: activityLog.createdAt }).from(activityLog).where(and( + eq(activityLog.companyId, companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), gte(activityLog.createdAt, since), + sql`${activityLog.details}->>'outcome' = 'queued'`, + sql`coalesce(${activityLog.details}->>'continuedFromSavedWork', '') <> 'true'`)).limit(64), + db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since))).limit(64), + freshContinuationTimes(db, companyId, issueId, since), + ]); + return [...plainContinuations.map(row => row.createdAt), ...retries.map(row => row.createdAt), ...fresh.map(record => record.at)] + .sort((a, b) => a.getTime() - b.getTime()); +} diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index d2dbce5c7b1..7bc2608bb8f 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -101,7 +101,8 @@ import { workspaceOperations, } from "@paperclipai/db"; import { parseObject, asBoolean, asNumber } from "../../adapters/utils.js"; -import { STRANDED_KIND, strandedRunCauseIsTemporary, type StrandedHoldRun } from "./stranded-hold-continuation.js"; +import { canContinueRecoveryRunAutomatically } from "../continuation-cause.js"; +import { STRANDED_CONTINUATION_MAX_AGE_MS, STRANDED_KIND, strandedRunCauseIsTemporary, type StrandedHoldRun } from "./stranded-hold-continuation.js"; import { runningProcesses } from "../../adapters/index.js"; import { isNativeRunnerOwnershipHeld, @@ -303,6 +304,10 @@ type LatestIssueRun = > & { resultJson?: unknown; rowVersion?: string; + finishedAt?: Date | null; + retryOfRunId?: string | null; + scheduledRetryReason?: string | null; + runnerProfileJson?: unknown; }) | null; type SuccessfulLatestIssueRun = NonNullable & { @@ -561,6 +566,19 @@ function didAutomaticRecoveryFail( ); } +function assignedWorkSweepOwnsFailedRun( + issue: Pick, + latestRun: LatestIssueRun, + agentId: string, +) { + return Boolean(latestRun && (issue.status === "todo" || issue.status === "in_progress") && !issue.assigneeUserId + && issue.assigneeAgentId === agentId && latestRun.agentId === agentId + && Date.now() - (latestRun.finishedAt ?? latestRun.createdAt).getTime() <= STRANDED_CONTINUATION_MAX_AGE_MS + && canContinueRecoveryRunAutomatically({ status: latestRun.status, errorCode: latestRun.errorCode, error: latestRun.error, + retryOfRunId: latestRun.retryOfRunId ?? null, scheduledRetryReason: latestRun.scheduledRetryReason ?? null, + contextSnapshot: latestRun.contextSnapshot as Record | null, runnerProfileJson: latestRun.runnerProfileJson })); +} + function isTerminalIssueRun(latestRun: LatestIssueRun) { if (!latestRun) return false; return TERMINAL_HEARTBEAT_RUN_STATUSES.has(latestRun.status); @@ -1040,6 +1058,10 @@ export function recoveryService( resultJson: recoveryRunResultColumn, startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt, + finishedAt: heartbeatRuns.finishedAt, + retryOfRunId: heartbeatRuns.retryOfRunId, + scheduledRetryReason: heartbeatRuns.scheduledRetryReason, + runnerProfileJson: heartbeatRuns.runnerProfileJson, }) .from(heartbeatRuns) .where( @@ -4317,6 +4339,10 @@ export function recoveryService( status: input.latestRun?.status ?? "", errorCode: input.latestRun?.errorCode ?? null, error: input.latestRun?.error ?? null, + retryOfRunId: input.latestRun?.retryOfRunId ?? null, + scheduledRetryReason: input.latestRun?.scheduledRetryReason ?? null, + contextSnapshot: (input.latestRun?.contextSnapshot ?? null) as Record | null, + runnerProfileJson: input.latestRun?.runnerProfileJson, executionRecovery: (parseObject(input.latestRun?.resultJson).executionRecovery ?? null) as StrandedHoldRun["executionRecovery"], }), recoveryOwner: @@ -5650,6 +5676,10 @@ export function recoveryService( } if (didAutomaticRecoveryFail(latestRun, "assignment_recovery")) { + if (assignedWorkSweepOwnsFailedRun(issue, latestRun, agentId)) { + result.skipped += 1; + continue; + } const updated = await escalateStrandedAssignedIssue({ issue, previousStatus: "todo", @@ -5685,6 +5715,7 @@ export function recoveryService( result.issueIds.push(issue.id); } else if ( dispatchOutcome.retryExhausted && + !assignedWorkSweepOwnsFailedRun(issue, latestRun, agentId) && !(await hasActiveExecutionPath(issue.companyId, issue.id, null)) ) { // Same exhaustion as the in_progress lane: the lost dispatch's @@ -5895,6 +5926,11 @@ export function recoveryService( // escalates it as before. preparationRetry = latestRun!.agentId === agentId && await failedRunOwnsPreparationRetry(issue.companyId, latestRun!.id); + const sweepOwnsFailure = !preparationRetry && assignedWorkSweepOwnsFailedRun(issue, latestRun, agentId); + if (classification.kind === "non_retryable" && sweepOwnsFailure) { + result.skipped += 1; + continue; + } if (classification.kind === "non_retryable" && !preparationRetry) { const updated = await escalateStrandedAssignedIssue({ issue, @@ -5919,6 +5955,10 @@ export function recoveryService( agentId, classification.errorCode, ); + if (consecutive >= classification.maxAttempts && sweepOwnsFailure) { + result.skipped += 1; + continue; + } if (consecutive >= classification.maxAttempts) { const updated = await escalateStrandedAssignedIssue({ issue, @@ -5969,6 +6009,7 @@ export function recoveryService( result.issueIds.push(issue.id); } else if ( (recoveryOutcome.retryExhausted || recoveryOutcome.continuationRefused || preparationRetry) && + !assignedWorkSweepOwnsFailedRun(issue, latestRun, agentId) && !(await hasActiveExecutionPath(issue.companyId, issue.id, null)) ) { // The failed run's bounded transient retries are all spent, so no diff --git a/server/src/services/recovery/stranded-hold-continuation.ts b/server/src/services/recovery/stranded-hold-continuation.ts index 8951a820fff..d23d67fcd4f 100644 --- a/server/src/services/recovery/stranded-hold-continuation.ts +++ b/server/src/services/recovery/stranded-hold-continuation.ts @@ -1,22 +1,8 @@ -import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "../reusable-sandbox-resume-error.js"; +import { continuesAutomatically } from "../continuation-cause.js"; export const STRANDED_CONTINUATION_MAX_AGE_MS = 24 * 60 * 60_000; export const STRANDED_KIND = "stranded_assigned_issue"; -const RESTART_CLASS_CODES = new Set([ - "process_lost", "server_shutdown_interrupted", "controller_lease_lost", "native_session_interrupted", - "native_runner_process_exited", "execution_finalization_deadline_exceeded", "local_runtime_api_listener_unavailable", -]); - -const TRANSIENT_SETUP_CODES = new Set([ - REUSABLE_SANDBOX_RESUME_FAILED_CODE, "remote_workspace_prepare_timeout", "environment_provider_unavailable", - "workspace_git_scan_timeout", "workspace_git_scan_saturated", "repository_catalog_unavailable", -]); - -const TRANSIENT_SANDBOX_CAUSES = new Set([ - "transport_timeout", "transport_closed", "sandbox_state_transition", "provider_rate_limited", "provider_unavailable", -]); - export type StrandedHoldRun = { agentId: string; status: string; @@ -24,6 +10,10 @@ export type StrandedHoldRun = { error: string | null; finishedAt: Date | null; executionRecovery: { kind?: unknown; providerWorkStarted?: unknown } | null; + retryOfRunId?: string | null; + scheduledRetryReason?: string | null; + contextSnapshot?: Record | null; + runnerProfileJson?: unknown; }; export function strandedHoldContinuesAutomatically( @@ -39,14 +29,11 @@ export function strandedHoldContinuesAutomatically( } export function strandedRunCauseIsTemporary( - run: Pick, + run: Pick, ): boolean { - if (!["failed", "timed_out", "interrupted", "cancelled"].includes(run.status)) return false; - const code = run.errorCode ?? ""; - if (RESTART_CLASS_CODES.has(code)) return true; - if (!TRANSIENT_SETUP_CODES.has(code)) return false; - if (run.executionRecovery?.kind !== "bootstrap" || run.executionRecovery.providerWorkStarted !== false) return false; - if (code !== REUSABLE_SANDBOX_RESUME_FAILED_CODE) return true; - const cause = /\(cause=([a-z_]+)/.exec(run.error ?? "")?.[1]; - return cause !== undefined && TRANSIENT_SANDBOX_CAUSES.has(cause); + return continuesAutomatically({ status: run.status, errorCode: run.errorCode, error: run.error, + retryOfRunId: run.retryOfRunId ?? null, scheduledRetryReason: run.scheduledRetryReason ?? null, + contextSnapshot: run.contextSnapshot ?? null, runnerProfileJson: run.runnerProfileJson, + resultJson: { executionRecovery: run.executionRecovery } }); } diff --git a/server/src/services/recovery/task-automatic-attempts.ts b/server/src/services/recovery/task-automatic-attempts.ts index f84b53bdb5a..20a2aa266f7 100644 --- a/server/src/services/recovery/task-automatic-attempts.ts +++ b/server/src/services/recovery/task-automatic-attempts.ts @@ -1,27 +1,31 @@ import { and, desc, eq, gte, inArray, sql } from "drizzle-orm"; import { activityLog, heartbeatRuns, type Db } from "@paperclipai/db"; +import { ASSIGNED_WORK_CONTINUED_ACTION, AUTOMATIC_RETRY_REASONS, SCHEDULER_TRANSIENT_RETRY_REASON, freshContinuationTimes } from "./continuation-ledger.js"; import { CONTINUATION_CAP, CONTINUATION_CAP_WINDOW_MS } from "./assigned-work-invariant.js"; -export const ASSIGNED_WORK_CONTINUED_ACTION = "issue.continued_automatically"; -export const SCHEDULER_TRANSIENT_RETRY_REASON = "transient_failure"; +export { ASSIGNED_WORK_CONTINUED_ACTION, SCHEDULER_TRANSIENT_RETRY_REASON }; export async function taskAutomaticAttempts(db: Db, companyId: string, issueId: string, at: Date) { const since = new Date(at.getTime() - CONTINUATION_CAP_WINDOW_MS); const [continuations, transientRetries] = await Promise.all([ - db.select({ createdAt: activityLog.createdAt }).from(activityLog).where(and( + db.select({ createdAt: activityLog.createdAt, outcome: sql`${activityLog.details}->>'outcome'` }).from(activityLog).where(and( eq(activityLog.companyId, companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId), eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), gte(activityLog.createdAt, since))) .orderBy(desc(activityLog.createdAt)).limit(64), db.select({ createdAt: heartbeatRuns.createdAt }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - inArray(heartbeatRuns.scheduledRetryReason, [SCHEDULER_TRANSIENT_RETRY_REASON]), gte(heartbeatRuns.createdAt, since))) + inArray(heartbeatRuns.scheduledRetryReason, AUTOMATIC_RETRY_REASONS), gte(heartbeatRuns.createdAt, since))) .orderBy(desc(heartbeatRuns.createdAt)).limit(64), ]); - const all = [...continuations, ...transientRetries].map(row => row.createdAt); - return { continuations: continuations.map(row => row.createdAt), all }; + const fresh = (await freshContinuationTimes(db, companyId, issueId, since)) + .filter(record => record.trigger !== "assigned_work").map(record => record.at); + const others = [...transientRetries.map(row => row.createdAt), ...fresh]; + const all = [...continuations.map(row => row.createdAt), ...others]; + const started = [...continuations.filter(row => row.outcome !== "not_started").map(row => row.createdAt), ...others]; + return { continuations: continuations.map(row => row.createdAt), all, started }; } export async function transientRetryWithinTaskCap(db: Db, companyId: string, issueId: string, at: Date) { const attempts = await taskAutomaticAttempts(db, companyId, issueId, at); - return attempts.continuations.length === 0 || attempts.all.length < CONTINUATION_CAP; + return attempts.continuations.length === 0 || attempts.started.length < CONTINUATION_CAP; }