diff --git a/server/src/__tests__/assigned-work-stalled-queued-run.test.ts b/server/src/__tests__/assigned-work-stalled-queued-run.test.ts new file mode 100644 index 00000000000..5b006a9c68b --- /dev/null +++ b/server/src/__tests__/assigned-work-stalled-queued-run.test.ts @@ -0,0 +1,252 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { activityLog, agents, agentWakeupRequests, companies, createDb, heartbeatRuns, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js"; +import { createHeartbeatQueue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake } from "../services/heartbeat/queue.js"; +import { mergeCoalescedContextSnapshot } from "../services/heartbeat/run-preparation.js"; +import { createWakeQueue } from "../modules/wake-queue/index.js"; +import { OLDEST_DISPATCH_SHARE, QUEUED_RUN_AGING_MS } from "../services/heartbeat/queued-run-dispatch-order.js"; +import { createStalledQueuedRunCancel } from "../services/heartbeat/stalled-queued-run.js"; +import { CHAT_CONTROL_RECOVERY_ADMISSION_KEY } from "../services/chat-control-recovery-stop.js"; +import { ASSIGNED_WORK_CONTINUED_ACTION, assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; +import { CONTINUATION_CAP_WINDOW_MS } from "../services/recovery/assigned-work-invariant.js"; +import { logger } from "../middleware/logger.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const MINUTE = 60_000; + +describe.skipIf(!support.supported)("a queued run the dispatcher keeps passing over", () => { + let database: Awaited>; + let db: ReturnType; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("stalled-queued-run-"); + db = createDb(database.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); + afterEach(() => { vi.restoreAllMocks(); }); + + async function seed() { + const prefix = `S${randomUUID().slice(0, 5)}`; + const [company] = await db.insert(companies).values({ name: "Stalled", issuePrefix: prefix }).returning(); + const companyId = company!.id; + const [agent] = await db.insert(agents).values({ companyId, name: "Builder", adapterType: "process", status: "idle" }).returning(); + const agentId = agent!.id; + let number = 0; + const task = async () => { + number += 1; + const [row] = await db.insert(issues).values({ companyId, title: `Task ${number}`, status: "in_progress", assigneeAgentId: agentId, responsibleUserId: "person-1", + issueNumber: number, identifier: `${prefix}-${number}`, createdAt: new Date(Date.now() - 5 * 60 * MINUTE), + updatedAt: new Date(Date.now() - 3 * 60 * MINUTE) }).returning(); + return row!; + }; + const issue = await task(); + const other = await task(); + + async function queuedRun(options: { ageMs?: number; person?: boolean; refused?: boolean; context?: Record } = {}) { + const createdAt = new Date(Date.now() - (options.ageMs ?? 2 * 60 * MINUTE)); + const reason = options.person ? "task_retry" : "issue_continuation_needed"; + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: options.person ? "on_demand" : "automation", + triggerDetail: options.person ? "manual" : "system", reason, payload: { issueId: issue.id }, status: "claimed", + requestedByActorType: options.person ? "user" : "system", requestedByActorId: options.person ? "person-1" : null, + claimedAt: createdAt, createdAt }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: options.person ? "on_demand" : "automation", + triggerDetail: options.person ? "manual" : "system", status: "queued", wakeupRequestId: wake!.id, createdAt, updatedAt: createdAt, + contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: reason, ...options.context } }).returning(); + if (options.refused !== false) { + await db.update(heartbeatRuns).set({ runnerProfileJson: { [CHAT_CONTROL_RECOVERY_ADMISSION_KEY]: { version: 1, phase: "lost" } } }) + .where(eq(heartbeatRuns.id, run!.id)); + } + await db.update(agentWakeupRequests).set({ runId: run!.id }).where(eq(agentWakeupRequests.id, wake!.id)); + return run!; + } + + async function otherStarts(count: number, startedAgoMs: number, status = "succeeded") { + for (let index = 0; index < count; index++) { + const startedAt = new Date(Date.now() - startedAgoMs + index * 1_000); + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "assignment", triggerDetail: "system", status, + createdAt: new Date(startedAt.getTime() - 1_000), startedAt, finishedAt: status === "running" ? null : new Date(startedAt.getTime() + 500), + contextSnapshot: { issueId: other.id, taskId: other.id, wakeReason: "issue_assigned" } }); + } + } + + const callbacks = createHeartbeatQueueTestDependencies(db); + callbacks.getAgent.mockImplementation(async id => (await db.select().from(agents).where(eq(agents.id, id)))[0] ?? null); + callbacks.instanceSettings.getExperimental.mockResolvedValue({} as never); + callbacks.setWakeupStatus.mockImplementation(async (id, status, values) => { + if (id) await db.update(agentWakeupRequests).set({ ...values, status }).where(eq(agentWakeupRequests.id, id)); + }); + const admission = createWakeQueue(db, { + resolveResponsibleUserId: async () => "person-1", + getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }), + resolveSessionBeforeForWakeup: async () => null, + wakeAdmissionHelpers: { filterZombieCoalesceTarget: callbacks.filterZombieCoalesceTarget, + mergeCoalescedContextSnapshot, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake }, + recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, + }); + callbacks.wakeQueue.createAdmissionTransactionScope.mockImplementation(admission.createAdmissionTransactionScope); + callbacks.wakeQueue.admitWakeBehindIssueExecution.mockImplementation(admission.admitWakeBehindIssueExecution); + const queue = createHeartbeatQueue(db, callbacks); + const cancel = createStalledQueuedRunCancel(callbacks); + const sweep = (beforeCancel?: () => Promise) => assignedWorkSweep(db, { + enqueueWakeup: (id, opts) => queue.enqueueWakeup(id, opts) as Promise<{ id: string } | null>, + cancelStalledQueuedRun: async scope => { await beforeCancel?.(); return cancel(scope); }, + }); + async function olderRun(createdAgoMs: number, startedAgoMs: number) { + const startedAt = new Date(Date.now() - startedAgoMs); + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", triggerDetail: "system", status: "succeeded", + createdAt: new Date(Date.now() - createdAgoMs), startedAt, finishedAt: new Date(startedAt.getTime() + 500), + contextSnapshot: { issueId: other.id, taskId: other.id, wakeReason: "issue_continuation_needed" } }); + } + const runsOfTask = async () => db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`)).orderBy(heartbeatRuns.createdAt); + const readRun = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!; + const continuations = async () => db.select().from(activityLog).where(and(eq(activityLog.entityId, issue.id), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION))); + return { companyId, agentId, issue, queuedRun, otherStarts, olderRun, queue, sweep, runsOfTask, readRun, continuations }; + } + + async function replacementOf(f: Awaited>, stalledId: string) { + const runs = await f.runsOfTask(); + return runs.filter(run => run.id !== stalledId); + } + + it("cancels an automatic run the claim keeps refusing and queues exactly one claimable replacement", async () => { + const f = await seed(); + const stalled = await f.queuedRun(); + await f.otherStarts(OLDEST_DISPATCH_SHARE + 1, QUEUED_RUN_AGING_MS / 2); + expect(await f.queue.claimQueuedRun(await f.readRun(stalled.id))).toBeNull(); + expect((await f.readRun(stalled.id)).status).toBe("queued"); + + const sweep = f.sweep(); + expect((await sweep.listSuspects({ companyId: f.companyId }, null, 50)).map(row => row.id)).toContain(f.issue.id); + const check = await sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id }); + expect(check).toMatchObject({ verdict: { kind: "violation", reason: "queued_run_stalled", runId: stalled.id }, outcome: "replaced_stalled_run" }); + + expect(await f.readRun(stalled.id)).toMatchObject({ status: "cancelled", errorCode: "queued_run_stalled" }); + const replacements = await replacementOf(f, stalled.id); + expect(replacements).toHaveLength(1); + const replacement = replacements[0]!; + expect(replacement.contextSnapshot).toMatchObject({ wakeReason: "issue_continuation_needed", previousRunId: stalled.id, + replacesStalledRunId: stalled.id }); + expect(replacement.retryOfRunId).toBeNull(); + const [entry] = await f.continuations(); + expect(entry?.details).toMatchObject({ outcome: "queued", violation: "queued_run_stalled" }); + expect(String((entry?.details as Record).message)).not.toMatch(/queued_run_stalled|[0-9a-f]{8}-/); + + const current = await f.readRun(replacement.id); + if (current.status === "queued") expect(await f.queue.claimQueuedRun(current)).not.toBeNull(); + expect((await f.readRun(replacement.id)).status).toBe("running"); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id })) + .toMatchObject({ verdict: { kind: "ok", clause: "live_run", runId: replacement.id } }); + }, 60_000); + + it("counts only the starts that passed the run over after it became its agent's oldest queued run", async () => { + const f = await seed(); + const waiting = await f.queuedRun(); + await f.olderRun(4 * 60 * MINUTE, 50 * MINUTE); + await f.otherStarts(OLDEST_DISPATCH_SHARE + 2, 55 * MINUTE); + await f.olderRun(3 * 60 * MINUTE, 20 * MINUTE); + const sweep = f.sweep(); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id })) + .toMatchObject({ verdict: { kind: "ok", clause: "live_run", runId: waiting.id } }); + await f.otherStarts(OLDEST_DISPATCH_SHARE, 10 * MINUTE); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id })) + .toMatchObject({ verdict: { kind: "ok", clause: "live_run", runId: waiting.id } }); + expect((await f.readRun(waiting.id)).status).toBe("queued"); + await f.otherStarts(1, 5 * MINUTE); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id })) + .toMatchObject({ verdict: { reason: "queued_run_stalled", runId: waiting.id }, outcome: "replaced_stalled_run" }); + }, 60_000); + + it("keeps the run when a person's message joins it after the check", async () => { + const f = await seed(); + const stalled = await f.queuedRun(); + await f.otherStarts(OLDEST_DISPATCH_SHARE + 1, QUEUED_RUN_AGING_MS / 2); + const joined = await f.sweep(async () => { + await f.queue.enqueueWakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", reason: "issue_commented", + payload: { issueId: f.issue.id }, requestedByActorType: "user", requestedByActorId: "person-1", + contextSnapshot: { issueId: f.issue.id, taskId: f.issue.id, wakeReason: "issue_commented" } }); + }).checkIssue({ companyId: f.companyId, issueId: f.issue.id }); + expect(joined).toMatchObject({ verdict: { reason: "queued_run_stalled" }, outcome: "changed" }); + expect((await f.readRun(stalled.id)).status).toBe("queued"); + expect(await f.runsOfTask()).toHaveLength(1); + expect(await f.continuations()).toHaveLength(0); + const personWakes = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.runId, stalled.id), + eq(agentWakeupRequests.requestedByActorType, "user"))); + expect(personWakes).toHaveLength(1); + }, 60_000); + + it("does nothing when the run starts after the check", async () => { + const f = await seed(); + const stalled = await f.queuedRun(); + await f.otherStarts(OLDEST_DISPATCH_SHARE + 1, QUEUED_RUN_AGING_MS / 2); + const started = await f.sweep(async () => { + await db.update(heartbeatRuns).set({ status: "running", startedAt: new Date() }).where(eq(heartbeatRuns.id, stalled.id)); + }).checkIssue({ companyId: f.companyId, issueId: f.issue.id }); + expect(started).toMatchObject({ outcome: "changed" }); + expect((await f.readRun(stalled.id)).status).toBe("running"); + expect(await f.runsOfTask()).toHaveLength(1); + expect(await f.continuations()).toHaveLength(0); + }, 60_000); + + it("leaves a run that waits behind a full agent", async () => { + const f = await seed(); + const waiting = await f.queuedRun(); + await f.otherStarts(1, QUEUED_RUN_AGING_MS * 3, "running"); + const sweep = f.sweep(); + expect((await sweep.listSuspects({ companyId: f.companyId }, null, 50)).map(row => row.id)).not.toContain(f.issue.id); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id })) + .toMatchObject({ verdict: { kind: "ok", clause: "live_run", runId: waiting.id }, outcome: "ok" }); + expect((await f.readRun(waiting.id)).status).toBe("queued"); + expect(await f.runsOfTask()).toHaveLength(1); + expect(await f.continuations()).toHaveLength(0); + }, 60_000); + + it("reports a person's refused Retry and never cancels it", async () => { + const f = await seed(); + const retry = await f.queuedRun({ person: true }); + await f.otherStarts(OLDEST_DISPATCH_SHARE + 1, QUEUED_RUN_AGING_MS / 2); + const warn = vi.spyOn(logger, "warn"); + const check = await f.sweep().checkIssue({ companyId: f.companyId, issueId: f.issue.id }); + expect(check).toMatchObject({ verdict: { kind: "violation", reason: "queued_run_stalled", runId: retry.id }, outcome: "left_person_work" }); + expect((await f.readRun(retry.id)).status).toBe("queued"); + expect(await f.runsOfTask()).toHaveLength(1); + expect(await f.continuations()).toHaveLength(0); + expect(warn.mock.calls.filter(([payload]) => (payload as { event?: string })?.event === "assigned_work_invariant.queued_run_stalled")) + .toHaveLength(1); + }, 60_000); + + it("leaves a replacement that stalls again, warning once and looking again after the long backoff", async () => { + const f = await seed(); + const original = randomUUID(); + const replacement = await f.queuedRun({ context: { previousRunId: original, replacesStalledRunId: original } }); + await f.otherStarts(OLDEST_DISPATCH_SHARE + 1, QUEUED_RUN_AGING_MS / 2); + const warn = vi.spyOn(logger, "warn"); + const sweep = f.sweep(); + const before = Date.now(); + const first = await sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id }); + const second = await sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id }); + for (const check of [first, second]) { + expect(check).toMatchObject({ verdict: { reason: "queued_run_stalled", runId: replacement.id }, outcome: "left_stalled_replacement" }); + expect(check!.retryAt!.getTime()).toBeGreaterThanOrEqual(before + CONTINUATION_CAP_WINDOW_MS); + } + expect((await f.readRun(replacement.id)).status).toBe("queued"); + expect(await f.runsOfTask()).toHaveLength(1); + expect(await f.continuations()).toHaveLength(0); + expect(warn.mock.calls.filter(([payload]) => (payload as { event?: string })?.event === "assigned_work_invariant.queued_run_stalled")) + .toHaveLength(1); + }, 60_000); + + it("produces one replacement when two servers sweep the task at once", async () => { + const f = await seed(); + const stalled = await f.queuedRun(); + await f.otherStarts(OLDEST_DISPATCH_SHARE + 1, QUEUED_RUN_AGING_MS / 2); + const checks = await Promise.all([f.sweep(), f.sweep()].map(sweep => sweep.checkIssue({ companyId: f.companyId, issueId: f.issue.id }))); + expect(checks.filter(check => check?.outcome === "replaced_stalled_run")).toHaveLength(1); + expect((await f.readRun(stalled.id)).status).toBe("cancelled"); + expect(await replacementOf(f, stalled.id)).toHaveLength(1); + expect((await f.continuations()).filter(entry => (entry.details as Record).outcome === "queued")).toHaveLength(1); + }, 60_000); +}); diff --git a/server/src/__tests__/helpers/heartbeat-queue.ts b/server/src/__tests__/helpers/heartbeat-queue.ts index bea3762cb23..dc3496919c7 100644 --- a/server/src/__tests__/helpers/heartbeat-queue.ts +++ b/server/src/__tests__/helpers/heartbeat-queue.ts @@ -28,9 +28,9 @@ export function createHeartbeatQueueTestDependencies(db: Db): QueueTestDependenc getAgent: vi.fn(async () => { throw new Error("Unexpected agent lookup"); }), options: {}, // Same compare-and-set contract as the run lifecycle: write only from one of fromStatuses. - setRunStatusFromLive: vi.fn(async (id, status, fromStatuses, patch) => { + setRunStatusFromLive: vi.fn(async (id, status, fromStatuses, patch, _report, condition) => { const [updated] = await db.update(heartbeatRuns).set({ ...patch, status }) - .where(and(eq(heartbeatRuns.id, id), inArray(heartbeatRuns.status, fromStatuses))).returning(); + .where(and(eq(heartbeatRuns.id, id), inArray(heartbeatRuns.status, fromStatuses), ...(condition ? [condition] : []))).returning(); if (updated) return { run: updated, updated: true as const }; const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)); return { run: current ?? null, updated: false as const }; diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 85387808006..6dc109a8501 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -2,6 +2,7 @@ import { registerHeldPreparationResumer } from "./interrupted-preparation-hold-s import { registerCompletedTurnCleanupResumer } from "./native-runtime/completed-turn-cleanup-hold-settlement.js"; import { endStorageOutageWaitOn, reportStorageOutageWaitsTo, runInOwnStorageScope } from "../storage/storage-unavailable.js"; import { createStorageWaitNotice } from "./heartbeat/storage-wait-notice.js"; +import { createStalledQueuedRunCancel } from "./heartbeat/stalled-queued-run.js"; import { resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js"; import { isProcessRunAdmissionClosed } from "./heartbeat/run-admission.js"; import { isYieldingRunStarts, refreshRunStartYieldIfStale, type RunStartHandover } from "./run-start-handover.js"; @@ -1041,6 +1042,7 @@ export function heartbeatService( // Bound at call time: the heartbeat recovery module is created below. settleLostRunProcess: scope => recoverActiveExecutions(scope), startSavedWorkRecovery: source => startOverdueWorkspaceRecovery(db, environmentRuntime, resumeRemoteStopComments, source), + cancelStalledQueuedRun: createStalledQueuedRunCancel({ setRunStatusFromLive, setWakeupStatus, appendRunEvent, releaseIssueExecutionAndPromote }), settleExplicitContinuationRetry: releaseIssueExecutionAndPromote, scheduleRecoveryRetry: async (runId, outcome) => { const [run] = await db diff --git a/server/src/services/heartbeat/queued-run-line.ts b/server/src/services/heartbeat/queued-run-line.ts index 097cf00fbd9..29d3c6bcd28 100644 --- a/server/src/services/heartbeat/queued-run-line.ts +++ b/server/src/services/heartbeat/queued-run-line.ts @@ -40,7 +40,7 @@ export function issueRunPriorityRank(priority: string | null | undefined) { } } -async function listPersonOriginSourceRunIds(db: Db, companyId: string, sourceRunIds: string[]) { +export async function listPersonOriginSourceRunIds(db: Db, companyId: string, sourceRunIds: string[]) { const uuidPattern = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; const result = await db.execute<{ root_id: string }>(sql` with recursive chain(root_id, run_id, depth) as ( diff --git a/server/src/services/heartbeat/stalled-queued-run.ts b/server/src/services/heartbeat/stalled-queued-run.ts new file mode 100644 index 00000000000..82b4865b01e --- /dev/null +++ b/server/src/services/heartbeat/stalled-queued-run.ts @@ -0,0 +1,32 @@ +import { sql } from "drizzle-orm"; +import type { HeartbeatQueueDependencies } from "./queue.js"; + +export const QUEUED_RUN_STALLED_CODE = "queued_run_stalled"; +export const QUEUED_RUN_STALLED_MESSAGE = + "Paperclip replaced this run with a new one: it had waited over an hour to start while its agent started other work."; + +type StalledRunLifecycle = Pick; + +export function stalledRunUnchangedCondition(fence: { version: string; attachedWakes: number }) { + return sql`"heartbeat_runs".xmin::text = ${fence.version} + and (select count(*)::int from agent_wakeup_requests attached where attached.company_id = "heartbeat_runs".company_id + and (attached.run_id = "heartbeat_runs".id or attached.id = "heartbeat_runs".wakeup_request_id)) = ${fence.attachedWakes}`; +} + +export function createStalledQueuedRunCancel(lifecycle: StalledRunLifecycle) { + return async function cancelStalledQueuedRun(scope: { companyId: string; runId: string; version: string; attachedWakes: number }): + Promise { + const finishedAt = new Date(); + const write = await lifecycle.setRunStatusFromLive(scope.runId, "cancelled", ["queued"], { + finishedAt, errorCode: QUEUED_RUN_STALLED_CODE, error: QUEUED_RUN_STALLED_MESSAGE, + }, undefined, stalledRunUnchangedCondition(scope)); + if (!write.updated || !write.run) return false; + const cancelled = write.run; + await lifecycle.setWakeupStatus(cancelled.wakeupRequestId, "cancelled", { finishedAt, error: QUEUED_RUN_STALLED_MESSAGE }); + await lifecycle.appendRunEvent(cancelled, { eventType: "lifecycle", stream: "system", level: "warn", + message: QUEUED_RUN_STALLED_MESSAGE, payload: { reason: QUEUED_RUN_STALLED_CODE } }); + await lifecycle.releaseIssueExecutionAndPromote(cancelled, { suppressImmediateRecovery: true }); + return true; + }; +} diff --git a/server/src/services/recovery/assigned-work-invariant.ts b/server/src/services/recovery/assigned-work-invariant.ts index 518287850bd..317f69c5516 100644 --- a/server/src/services/recovery/assigned-work-invariant.ts +++ b/server/src/services/recovery/assigned-work-invariant.ts @@ -69,6 +69,7 @@ export const VIOLATION_REASONS = [ * recovery queue has not started that recovery within its bound. Repaired * by starting the recovery, never by a wake its hold would refuse. */ "saved_work_recovery_overdue", + "queued_run_stalled", ] as const; export type ViolationReason = (typeof VIOLATION_REASONS)[number]; @@ -141,7 +142,8 @@ export type AssignedWorkFacts = { awaitingFirstMessage: boolean; }; /** Non-terminal runs (queued, running, scheduled_retry) for this task. */ - activeRuns: Array<{ id: string; status: string; scheduledRetryAt: Date | null; owner: RunOwnerVerdict }>; + activeRuns: Array<{ id: string; status: string; scheduledRetryAt: Date | null; owner: RunOwnerVerdict; + queuedStall?: { personWork: boolean; replacement: boolean; queuedSince: Date } | null }>; latestRun: { id: string; status: string; @@ -228,8 +230,10 @@ export function evaluateAssignedWork(facts: AssignedWorkFacts, now: Date = new D // 1. A live run with an owner that is not proven gone. An unprovable owner // fails closed: during a deploy overlap another server may still own it. + const stalledQueued = issue.status === "blocked" ? undefined + : facts.activeRuns.find(run => run.status === "queued" && run.queuedStall); const live = facts.activeRuns.find(run => (run.status === "queued" || run.status === "running") - && run.owner !== "gone" && run.owner !== "stalled"); + && run.owner !== "gone" && run.owner !== "stalled" && !(stalledQueued && run.queuedStall)); if (live) { if (issue.status === "blocked" && !waitingReason(facts, now)) { return { kind: "violation", reason: "blocked_with_live_run", runId: live.id, stale: false }; @@ -245,7 +249,7 @@ export function evaluateAssignedWork(facts: AssignedWorkFacts, now: Date = new D if (stalled) return { kind: "violation", reason: "native_run_stalled", runId: stalled.id, stale: false }; // 2. A durable intent that starts a run without anyone acting. - const intent = pendingIntent(facts, now); + const intent = pendingIntent(facts, now, Boolean(stalledQueued)); if (intent) return { kind: "ok", clause: "pending_intent", intent }; // A run that ended moments ago is still settling: its own failure path may @@ -261,6 +265,8 @@ export function evaluateAssignedWork(facts: AssignedWorkFacts, now: Date = new D // them in time. A continuation wake would only wait behind the same hold. if (facts.savedWorkRecovery) return { kind: "violation", reason: "saved_work_recovery_overdue", stale: false }; + if (stalledQueued) return { kind: "violation", reason: "queued_run_stalled", runId: stalledQueued.id, stale: false }; + if (issue.status === "blocked") { const blockedAt = issue.blockedTransitionAt ?? issue.updatedAt; const stale = now.getTime() - blockedAt.getTime() > STALE_BLOCK_MS; @@ -275,7 +281,7 @@ export function evaluateAssignedWork(facts: AssignedWorkFacts, now: Date = new D return { kind: "violation", reason: "no_pending_path", stale: false }; } -function pendingIntent(facts: AssignedWorkFacts, now: Date): PendingIntent | null { +function pendingIntent(facts: AssignedWorkFacts, now: Date, queuedRunStalled = false): PendingIntent | null { if (facts.pendingWakes > 0) return "queued_wake"; if (facts.activeRuns.some(run => run.status === "scheduled_retry")) return "scheduled_retry"; if (facts.nativeRecoveryPending) return "native_recovery"; @@ -285,7 +291,7 @@ function pendingIntent(facts: AssignedWorkFacts, now: Date): PendingIntent | nul if (facts.savedWorkRecovery && savedWorkRecoveryPending(facts.savedWorkRecovery, now)) return "saved_work_recovery"; const interruption = facts.interruption; if (interruption?.responsibility === "system") { - if (interruption.state === "working" && !(facts.latestRun?.finishedAt + if (interruption.state === "working" && !queuedRunStalled && !(facts.latestRun?.finishedAt && now.getTime() - facts.latestRun.finishedAt.getTime() > SYSTEM_RECOVERY_WORKING_BOUND_MS)) return "system_recovery_working"; if (interruption.state === "retry_scheduled" && interruption.nextRetryAt && Date.parse(interruption.nextRetryAt) > now.getTime()) return "scheduled_retry"; @@ -395,6 +401,8 @@ export function continuationActivityMessage(reason: ViolationReason): string { return "No longer blocked: this task is already queued to continue, so nothing needs you."; case "saved_work_recovery_overdue": return "Paperclip started recovering this task's saved files. The task continues once they are recovered."; + case "queued_run_stalled": + return "Continued automatically: this task had waited over an hour to start while its agent started other work, so Paperclip queued it again."; default: return "Continued automatically after an interruption. Nothing was running or scheduled, so Paperclip queued the task again."; } diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index c5c34a68819..880445c9921 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -31,6 +31,8 @@ import { CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js"; import { workspaceRecoverySourceEligible } from "../workspace-recovery-source.js"; import { PENDING_CONTINUATION_BOUND_MS } from "../recovery-continuation-pacing.js"; +import { OLDEST_DISPATCH_SHARE, QUEUED_RUN_AGING_MS } from "../heartbeat/queued-run-dispatch-order.js"; +import { listPersonOriginSourceRunIds } from "../heartbeat/queued-run-line.js"; import { attributeBlock, BLOCK_ATTRIBUTION_WINDOW_MS, blockedStatusDetailsSql } from "../interruption-block-release.js"; import { ASSIGNED_WORK_CONTINUED_ACTION, taskAutomaticAttempts } from "./task-automatic-attempts.js"; import { isReattachPhaseActive, reattachPhaseRecheckAt } from "../heartbeat/run-admission.js"; @@ -45,6 +47,7 @@ import { savedWorkRecoveryDueSince, CONTINUATION_REFUSED_MESSAGE, CONTINUATION_REVIEW_MESSAGE, + CONTINUATION_CAP_WINDOW_MS, evaluateAssignedWork, STALE_BLOCK_MS, NATIVE_STALL_MS, @@ -94,6 +97,7 @@ export type AssignedWorkDeps = { /** Starts a saved-work recovery the recovery queue has not reached in time. */ startSavedWorkRecovery?: (source: { companyId: string; issueId: string; runId: string; actionId: string }) => Promise<"started" | "in_flight" | "busy" | "not_eligible">; + cancelStalledQueuedRun?: (scope: { companyId: string; runId: string; version: string; attachedWakes: number }) => Promise; /** Shared owner proof, injectable for tests. */ proveOwner?: (stamp: LocalOwnerStamp) => Promise; now?: () => Date; @@ -102,10 +106,13 @@ export type AssignedWorkDeps = { export type AssignedWorkCheck = { verdict: AssignedWorkVerdict; outcome: "ok" | "settling" | "repaired" | "settled_lost_run" | "deferred_rate_limit" | "deferred_cap" | "deferred_reattach_phase" | "stale_reported" | "left_to_hold_owner" | "left_to_native_recovery" | "changed" | "enqueue_refused" - | "waiting_on_recovery" | "started_saved_work_recovery" | "saved_work_recovery_busy"; + | "waiting_on_recovery" | "started_saved_work_recovery" | "saved_work_recovery_busy" + | "replaced_stalled_run" | "left_person_work" | "left_stalled_replacement"; retryAt: Date | null; }; +const rowsOf = (value: unknown): T[] => Array.isArray(value) ? value as T[] : ((value as { rows?: T[] })?.rows ?? []); + const json = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; @@ -133,7 +140,8 @@ function createPassLog() { } else if (verdict.kind === "violation") { pass.violations[verdict.reason] = (pass.violations[verdict.reason] ?? 0) + 1; } - if (check.outcome === "repaired" || check.outcome === "settled_lost_run" || check.outcome === "started_saved_work_recovery") pass.repaired += 1; + if (check.outcome === "repaired" || check.outcome === "settled_lost_run" || check.outcome === "started_saved_work_recovery" + || check.outcome === "replaced_stalled_run") pass.repaired += 1; if (check.outcome === "deferred_rate_limit" || check.outcome === "deferred_cap") pass.deferred += 1; if (check.outcome === "stale_reported") pass.stale += 1; if (!timer) { timer = setTimeout(flush, 5_000); timer.unref?.(); } @@ -147,6 +155,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { const proveOwner = deps.proveOwner ?? ((stamp: LocalOwnerStamp) => proveOwnerGone(stamp, { ...currentProcessProbe(), readOnly: true })); const passLog = createPassLog(); const repairTimes: number[] = []; + const reportedStalledRuns = new Set(); const sweepTreeControl = issueTreeControlService(db); async function runOwner(run: RunRow, at: Date, proveOwners = true): Promise { @@ -194,8 +203,11 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { const [activeRuns, latestRuns, wakes, chatWakes, nativePending, handoffs, goals, interactions, approvalRows, blockerRows, resolvedBlockerRows, childRows, actionMap, agentRows, companyRows, onboardingRows, blockActivity, interruptions, savedWorkRows] = await Promise.all([ - database.select(runColumns).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), - inArray(heartbeatRuns.status, NON_TERMINAL_RUN_STATUSES), sql`${runIssue} = any(${idParam})`)), + database.select({ ...runColumns, createdAt: heartbeatRuns.createdAt, stalledQueued: queuedRunStalledSql(aliasedRun("heartbeat_runs"), at), + replacesStalledRunId: sql`${heartbeatRuns.contextSnapshot}->>'replacesStalledRunId'`, + explicitUserContinuation: sql`jsonb_typeof(${heartbeatRuns.contextSnapshot}->'explicitUserContinuation') = 'object'` }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, NON_TERMINAL_RUN_STATUSES), sql`${runIssue} = any(${idParam})`)), // The newest run per task, read through the (company, issueId, created) index. database.execute(sql`select requested.id as "issueId", latest.* from unnest(${idParam}) as requested(id) @@ -219,6 +231,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { or (${agentWakeupRequests.status} = 'claimed' and (${agentWakeupRequests.runId} is not null or coalesce(${agentWakeupRequests.claimedAt}, ${agentWakeupRequests.createdAt}) > ${at.toISOString()}::timestamptz - ${`${DEFERRED_WAKE_PENDING_BOUND_MS} milliseconds`}::interval)))`, + sql`not exists (select 1 from heartbeat_runs wake_run where wake_run.id = ${agentWakeupRequests.runId} + and ${queuedRunStalledSql(aliasedRun("wake_run"), at)})`, sql`${agentWakeupRequests.payload}->>'issueId' = any(${idParam})`)) .groupBy(sql`${agentWakeupRequests.payload}->>'issueId'`), database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, ids), @@ -302,7 +316,6 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { .orderBy(desc(issueRecoveryActions.createdAt)), ]); - const rowsOf = (value: unknown): T[] => Array.isArray(value) ? value as T[] : ((value as { rows?: T[] })?.rows ?? []); const strandedRunIds = [...actionMap.values()].filter(action => action.kind === STRANDED_KIND && action.ownerType === "board") .map(action => action.evidence?.latestRunId).filter((id): id is string => typeof id === "string" && /^[0-9a-f-]{36}$/i.test(id)); const strandedRuns = new Map((strandedRunIds.length ? await database.select({ id: heartbeatRuns.id, @@ -366,11 +379,16 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { const blockActivityByIssue = new Map(rowsOf<{ issueId: string; actorType: string; actorId: string; agentId: string | null; details: unknown }>(blockActivity) .map(row => [row.issueId, row])); + const stalledQueuedIds = activeRuns.filter(row => row.stalledQueued).map(row => row.id); + const personQueuedIds = new Set(stalledQueuedIds.length ? await listPersonOriginSourceRunIds(database, companyId, stalledQueuedIds) : []); const ownedRunsByIssue = new Map(); for (const issue of issueRows) { const ownedRuns: AssignedWorkFacts["activeRuns"] = []; - for (const run of activeRuns.filter(row => row.issueId === issue.id) as RunRow[]) { - ownedRuns.push({ id: run.id, status: run.status, scheduledRetryAt: run.scheduledRetryAt, owner: stalledRunIds.has(run.id) && run.status === "running" ? "stalled" : await runOwner(run, at, proveOwners) }); + for (const run of activeRuns.filter(row => row.issueId === issue.id)) { + ownedRuns.push({ id: run.id, status: run.status, scheduledRetryAt: run.scheduledRetryAt, + owner: stalledRunIds.has(run.id) && run.status === "running" ? "stalled" : await runOwner(run as RunRow, at, proveOwners), + queuedStall: run.stalledQueued ? { personWork: personQueuedIds.has(run.id) || Boolean(run.explicitUserContinuation), + replacement: Boolean(run.replacesStalledRunId), queuedSince: new Date(run.createdAt) } : null }); } ownedRunsByIssue.set(issue.id, ownedRuns); } @@ -474,6 +492,12 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { and r.status in ('queued', 'running', 'scheduled_retry') and not (r.status = 'running' and r.runtime_mode = 'legacy' and r.controller_lease_expires_at is not null and r.controller_lease_expires_at < ${at.toISOString()}::timestamptz))`; + const liveUnstalledRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} + and (r.context_snapshot->>'issueId') = ${issues.id}::text + and r.status in ('queued', 'running', 'scheduled_retry') + and not (r.status = 'running' and r.runtime_mode = 'legacy' and r.controller_lease_expires_at is not null + and r.controller_lease_expires_at < ${at.toISOString()}::timestamptz) + and not ${queuedRunStalledSql(aliasedRun("r"), at)})`; return db.select({ id: issues.id, companyId: issues.companyId, updatedAt: issues.updatedAt }).from(issues).where(and( scope.companyId ? eq(issues.companyId, scope.companyId) : undefined, scope.issueId ? eq(issues.id, scope.issueId) : undefined, @@ -489,13 +513,15 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { sql`(${issues.monitorNextCheckAt} is null or ${issues.monitorNextCheckAt} <= ${at.toISOString()}::timestamptz)`, // A legacy run whose controller lease lapsed may have lost its owner; // the predicate judges it with the owner proof. Other active runs are live. - sql`(${issues.status} = 'blocked' or not ${liveRun})`, + sql`(${issues.status} = 'blocked' or not ${liveUnstalledRun})`, sql`((${issues.status} = 'blocked' and ${liveRun}) or not exists (select 1 from agent_wakeup_requests w where w.company_id = ${issues.companyId} and (w.payload->>'issueId') = ${issues.id}::text and (w.status = 'queued' or (w.status = 'deferred_issue_execution' and w.created_at > ${new Date(at.getTime() - DEFERRED_WAKE_PENDING_BOUND_MS).toISOString()}::timestamptz) or (w.status = 'claimed' and (w.run_id is not null - or coalesce(w.claimed_at, w.created_at) > ${new Date(at.getTime() - DEFERRED_WAKE_PENDING_BOUND_MS).toISOString()}::timestamptz)))))`, + or coalesce(w.claimed_at, w.created_at) > ${new Date(at.getTime() - DEFERRED_WAKE_PENDING_BOUND_MS).toISOString()}::timestamptz))) + and not exists (select 1 from heartbeat_runs wake_run where wake_run.id = w.run_id + and ${queuedRunStalledSql(aliasedRun("wake_run"), at)})))`, sql`not exists (select 1 from issue_thread_interactions t where t.company_id = ${issues.companyId} and t.issue_id = ${issues.id} and t.status = 'pending' and ${interactionNotSupersededSql(sql`t.company_id`, sql`t.issue_id::text`, sql`t.created_at`)})`, @@ -573,6 +599,14 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { await deps.settleLostRunProcess?.({ companyId: issue.companyId, runId: verdict.runId! }); return { verdict, outcome: "settled_lost_run", retryAt: new Date(at.getTime() + 90_000) }; } + if (verdict.reason === "queued_run_stalled") { + const stall = facts.activeRuns.find(run => run.id === verdict.runId)?.queuedStall ?? null; + if (!stall || stall.personWork || stall.replacement || !deps.cancelStalledQueuedRun) { + reportStalledRun(issue, verdict.runId!, stall, at); + return { verdict, outcome: stall?.personWork ? "left_person_work" : "left_stalled_replacement", + retryAt: new Date(at.getTime() + CONTINUATION_CAP_WINDOW_MS) }; + } + } // A small process-wide budget keeps a post-deploy backlog from starting // hundreds of runs at once. Deferred tasks come back on their deadline. const minuteAgo = at.getTime() - 60_000; @@ -584,6 +618,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { if (claimed.kind !== "claimed") return { verdict, outcome: claimed.kind === "capped" ? "deferred_cap" : claimed.kind === "changed" ? "changed" : "deferred_rate_limit", retryAt: claimed.kind === "changed" ? null : claimed.retryAt }; repairTimes.push(at.getTime()); + if (claimed.reason === "queued_run_stalled") return replaceStalledRun(issue, claimed, claimed.verdict.runId!); // A block that interrupted a review goes back to review; the review's own // recovery wakes its participant. No agent work starts from here. if (claimed.restoredTo === "in_review") { @@ -621,6 +656,40 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { return { verdict: claimed.verdict, outcome: run ? "repaired" : "enqueue_refused", retryAt: claimed.nextCheckAt }; } + function reportStalledRun(issue: IssueRow, runId: string, stall: NonNullable | null, at: Date) { + if (reportedStalledRuns.has(runId)) return; + if (reportedStalledRuns.size >= 1_000) reportedStalledRuns.clear(); + reportedStalledRuns.add(runId); + logger.warn({ event: "assigned_work_invariant.queued_run_stalled", companyId: issue.companyId, issueId: issue.id, runId, + agentId: issue.assigneeAgentId, queuedForMs: stall ? at.getTime() - stall.queuedSince.getTime() : null, + personWork: stall?.personWork ?? null, replacement: stall?.replacement ?? null }, + "queued run its agent keeps passing over was left in place"); + } + + async function replaceStalledRun(issue: IssueRow, claimed: Claimed, runId: string): Promise { + const cancelled = !claimed.stalledFence ? false : await deps.cancelStalledQueuedRun!({ companyId: issue.companyId, runId, + ...claimed.stalledFence }).catch(error => { + logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id, runId }, "assigned-work sweep could not cancel a stalled queued run"); + return false; + }); + if (!cancelled) { + await db.delete(activityLog).where(eq(activityLog.id, claimed.activityId)); + return { verdict: claimed.verdict, outcome: "changed", retryAt: null }; + } + const run = await deps.enqueueWakeup(issue.assigneeAgentId!, { + source: "automation", triggerDetail: "system", reason: "issue_continuation_needed", + payload: { issueId: issue.id, mutation: "assigned_work_invariant" }, + requestedByActorType: "system", requestedByActorId: null, + contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: "issue_continuation_needed", + source: ASSIGNED_WORK_CONTINUATION_SOURCE, assignedWorkViolation: claimed.reason, previousRunId: runId, replacesStalledRunId: runId }, + }).catch(error => { + logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id }, "assigned-work replacement for a stalled run was not admitted"); + return null; + }); + await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started"); + return { verdict: claimed.verdict, outcome: run ? "replaced_stalled_run" : "enqueue_refused", retryAt: claimed.nextCheckAt }; + } + async function continueFromSavedWork(issue: IssueRow, facts: AssignedWorkFacts, at: Date): Promise { const source = facts.latestRun; if (!source?.finishedAt || !["failed", "timed_out", "interrupted", "cancelled"].includes(source.status) @@ -660,7 +729,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { type Claimed = { kind: "claimed"; nextCheckAt: Date; activityId: string; publications: ActivityPublication[]; verdict: Extract; reason: ViolationReason; restoredTo: "todo" | "in_progress" | "in_review" | null; restoredStatusVersion: number | null; - blockedTransitionAt: Date | null; unblockDescriptor: IssueRow["unblockDescriptor"]; blockedOwnerNotifiedAt: Date | null }; + blockedTransitionAt: Date | null; unblockDescriptor: IssueRow["unblockDescriptor"]; blockedOwnerNotifiedAt: Date | null; + stalledFence: { version: string; attachedWakes: number } | null }; /** * The durable, serialized claim. Under the task's row lock it loads the @@ -679,10 +749,22 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { .where(and(eq(issues.companyId, issue.companyId), eq(issues.id, issue.id))).for("update"); if (!current || current.status !== issue.status || current.statusVersion !== issue.statusVersion || current.assigneeAgentId !== issue.assigneeAgentId || current.assigneeUserId || current.hiddenAt) return { kind: "changed" as const }; + const fences = rowsOf<{ id: string; version: string; attachedWakes: number }>(await tx.execute(sql` + select r.id, r.xmin::text as version, (select count(*)::int from agent_wakeup_requests w + where w.company_id = r.company_id and (w.run_id = r.id or w.id = r.wakeup_request_id)) as "attachedWakes" + from heartbeat_runs r where r.company_id = ${issue.companyId} and r.status = 'queued' + and (r.context_snapshot->>'issueId') = ${issue.id}`)); const fresh = (await loadFacts(issue.companyId, [current], { database: tx as unknown as Db, proveOwners: false })).get(current.id)!; const verdict = evaluateAssignedWork(fresh, at); + let stalledFence: Claimed["stalledFence"] = null; if (verdict.kind !== "violation" || verdict.stale || verdict.reason === "run_owner_gone" || verdict.reason === "native_run_stalled" || (current.status === "blocked" && fresh.activeRecoveryAction && !fresh.activeRecoveryAction.continuedBySweep)) return { kind: "changed" as const }; + if (verdict.reason === "queued_run_stalled") { + const stall = fresh.activeRuns.find(run => run.id === verdict.runId)?.queuedStall; + const fence = fences.find(row => row.id === verdict.runId); + if (!stall || stall.personWork || stall.replacement || !deps.cancelStalledQueuedRun || !fence) return { kind: "changed" as const }; + stalledFence = { version: fence.version, attachedWakes: Number(fence.attachedWakes) }; + } const attempts = await taskAutomaticAttempts(tx as unknown as Db, issue.companyId, issue.id, at); const recent = attempts.continuations; const allowance = continuationAllowance(attempts.started, at, attempts.all); @@ -717,7 +799,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { attempt: recent.length + 1 }, }, publications); const next = continuationAllowance([at, ...attempts.started], at, [at, ...attempts.all]); - return { kind: "claimed" as const, activityId: activity.id, publications, verdict, reason: verdict.reason, + return { kind: "claimed" as const, activityId: activity.id, publications, verdict, reason: verdict.reason, stalledFence, restoredTo, restoredStatusVersion, blockedTransitionAt: current.blockedTransitionAt, unblockDescriptor: current.unblockDescriptor, blockedOwnerNotifiedAt: current.blockedOwnerNotifiedAt, nextCheckAt: next.allowed ? new Date(at.getTime() + 90_000) : next.retryAt }; @@ -862,6 +944,25 @@ function interactionNotSupersededSql(companyId: SQLWrapper, issueIdText: SQLWrap and retry.created_at > ${createdAt})`; } +type StalledRunColumns = { id: SQLWrapper; companyId: SQLWrapper; agentId: SQLWrapper; status: SQLWrapper; createdAt: SQLWrapper }; + +function queuedRunStalledSql(run: StalledRunColumns, at: Date) { + const agedBefore = new Date(at.getTime() - QUEUED_RUN_AGING_MS).toISOString(); + return sql`(${run.status} = 'queued' and ${run.createdAt} <= ${agedBefore}::timestamptz + and not exists (select 1 from heartbeat_runs older where older.company_id = ${run.companyId} + and older.agent_id = ${run.agentId} and older.status = 'queued' and older.created_at < ${run.createdAt}) + and (select count(*) from (select 1 from heartbeat_runs started where started.company_id = ${run.companyId} + and started.agent_id = ${run.agentId} and started.id <> ${run.id} + and started.started_at > greatest(${run.createdAt} + ${`${QUEUED_RUN_AGING_MS} milliseconds`}::interval, + (select max(coalesce(prior.started_at, prior.finished_at)) from heartbeat_runs prior + where prior.company_id = ${run.companyId} and prior.agent_id = ${run.agentId} and prior.created_at < ${run.createdAt} + and (prior.started_at >= ${run.createdAt} or (prior.started_at is null and prior.finished_at >= ${run.createdAt})))) + limit ${OLDEST_DISPATCH_SHARE + 1}) passed_over) > ${OLDEST_DISPATCH_SHARE})`; +} + +const aliasedRun = (alias: string): StalledRunColumns => ({ id: sql.raw(`${alias}.id`), companyId: sql.raw(`${alias}.company_id`), + agentId: sql.raw(`${alias}.agent_id`), status: sql.raw(`${alias}.status`), createdAt: sql.raw(`${alias}.created_at`) }); + /** The status a block replaced, from the activity entry that blocked the task: * the normal update route records it as `_previous.status`, recovery as * `previousStatus`, the native status committer as `fromStatus`. */ diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 7bc2608bb8f..8b9a3ad4a5f 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -1016,6 +1016,7 @@ export function recoveryService( settleLostRunProcess?: (scope: { companyId: string; runId: string; onDeferredDeadline?: (at: number) => void }) => Promise; /** Starts a saved-work recovery the recovery queue has not reached in time. */ startSavedWorkRecovery?: AssignedWorkDeps["startSavedWorkRecovery"]; + cancelStalledQueuedRun?: AssignedWorkDeps["cancelStalledQueuedRun"]; beforeOrphanedRunTerminalWrite?: (runId: string) => Promise; beforeStaleLockClear?: (issueId: string) => Promise; afterStrandedLatestRunRead?: (input: { issueId: string; runId: string | null }) => Promise; @@ -6902,6 +6903,7 @@ export function recoveryService( liveRunExecutions: deps.liveRunExecutions, settleLostRunProcess: deps.settleLostRunProcess, startSavedWorkRecovery: deps.startSavedWorkRecovery, + cancelStalledQueuedRun: deps.cancelStalledQueuedRun, }); return {