diff --git a/server/src/__tests__/heartbeat-provider-limit-wait.test.ts b/server/src/__tests__/heartbeat-provider-limit-wait.test.ts index 66067a46441..551bee138ef 100644 --- a/server/src/__tests__/heartbeat-provider-limit-wait.test.ts +++ b/server/src/__tests__/heartbeat-provider-limit-wait.test.ts @@ -11,6 +11,23 @@ import { createProviderLimitWait } from "../services/heartbeat/provider-limit-wa import { executionProjectionsForRuns } from "../services/execution-projection.js"; import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; +const providerLimitCancellation = vi.hoisted(() => ({ after: null as null | (() => Promise) })); +vi.mock("../services/heartbeat/provider-limit-wait.js", async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + createProviderLimitWait: (...[database, deps]: Parameters) => + actual.createProviderLimitWait(database, { + ...deps, + setRunStatusIfRunning: async (...args) => { + const result = await deps.setRunStatusIfRunning(...args); + await providerLimitCancellation.after?.(); + return result; + }, + }), + }; +}); + const describePostgres = recoveryPostgresSupport.supported ? describe : describe.skip; if (!recoveryPostgresSupport.supported) console.warn(`Skipping provider limit wait tests: ${recoveryPostgresSupport.reason}`); @@ -132,6 +149,25 @@ describePostgres("heartbeat provider usage limit wait", () => { expect(resumed.status).toBe("succeeded"); }, 60_000); + it("keeps the task lock for its retry when recovery sweeps the task right after the cancellation", async () => { + const f = await seed([rateLimited(16)]); + const sweeps: Array>> = []; + providerLimitCancellation.after = async () => { + sweeps.push(await heartbeat.sweepStaleIssueLocks({ companyId: f.companyId, issueId: f.issueId })); + }; + try { + const finished = await wake(f); + expect(finished).toMatchObject({ status: "cancelled", errorCode: "provider_limit_wait" }); + expect(sweeps).toEqual([expect.objectContaining({ cleared: 0 })]); + const [retry] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, finished.id)); + expect(retry).toMatchObject({ status: "scheduled_retry", scheduledRetryReason: "ai_connection_pool_wait" }); + const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + expect(issue.executionRunId).toBe(retry.id); + } finally { + providerLimitCancellation.after = null; + } + }, 60_000); + it("releases the task when its retry cannot be scheduled, even if diagnostics fail", async () => { const f = await seed([rateLimited(16)]); const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); diff --git a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts index 535c1c9de6a..ec74dd0e310 100644 --- a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts +++ b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts @@ -229,6 +229,23 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => { ); }); + it.each(["in_progress", "done"] as const)("waits for a live executor to settle its run before clearing the lock of a %s task", async status => { + const { companyId, agentId, failedRunId } = await seed(); + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Executor still settling", status, + priority: "high", assigneeAgentId: agentId, executionRunId: failedRunId, executionLockedAt: new Date() }); + const deadlines: number[] = []; + const whileLive = await recoveryService(db, { enqueueWakeup: vi.fn(), liveRunExecutions: new Set([failedRunId]) }) + .sweepStaleIssueLocks({ companyId, issueId, onDeferredDeadline: at => deadlines.push(at) }); + expect(whileLive.issueIds).toEqual(status === "done" ? [issueId] : []); + expect(deadlines).toHaveLength(status === "done" ? 0 : 1); + const afterExecutor = await recoveryService(db, { enqueueWakeup: vi.fn(), liveRunExecutions: new Set() }) + .sweepStaleIssueLocks({ companyId, issueId }); + expect(afterExecutor.issueIds).toEqual(status === "done" ? [] : [issueId]); + const [row] = await db.select({ executionRunId: issues.executionRunId }).from(issues).where(eq(issues.id, issueId)); + expect(row.executionRunId).toBeNull(); + }); + it("does not clear locks while the referenced run is still running", async () => { const { companyId, agentId, runningRunId } = await seed(); const issueId = randomUUID(); diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index cd7bdb1c7b1..513d1b1ed10 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -6883,6 +6883,17 @@ export function recoveryService( .some(runId => runId !== null && pendingResumeRunIds.has(runId))) { continue; } + if (!taskClosed) { + let settlingUntil: Date | null = null; + for (const runId of new Set([issue.checkoutRunId, issue.executionRunId])) { + const lockRun = runId ? runById.get(runId) : undefined; + if (lockRun && !settlingUntil) settlingUntil = await terminalRunSettlingUntil(issue.companyId, lockRun.id, lockRun); + } + if (settlingUntil) { + opts?.onDeferredDeadline?.(settlingUntil.getTime()); + continue; + } + } await deps.beforeStaleLockClear?.(issue.id); const updated = await db.transaction(async tx => {