diff --git a/docs/api/agents.md b/docs/api/agents.md index 7c5fb5f9bfa..511caee35bc 100644 --- a/docs/api/agents.md +++ b/docs/api/agents.md @@ -137,6 +137,27 @@ POST /api/agents/{agentId}/heartbeat/invoke Manually triggers a heartbeat for the agent. +## Assigned-Work Invariant Report + +``` +GET /api/companies/{companyId}/assigned-work-invariant +``` + +Board only. Read-only. Classifies every open task assigned to an agent (`todo`, `in_progress`, `blocked`): it has a live run, a pending intent that will start one, a named wait that is someone else's move, or it is a violation. Paperclip continues violations automatically; after a deploy, `violationCount` should return to 0. + +The response contains: + +- `checked`: the number of tasks classified, newest-updated first. At most 2,000. +- `openTasks`: the number of open agent-assigned tasks the caller may read. When it exceeds 2,000, `truncated` and `violationCountIsLowerBound` are `true`: the counts cover only the 2,000 newest tasks, so the real violation count can be higher, never lower. +- Only tasks the caller may read are counted; private tasks the caller cannot read are left out of every count and sample. On an instance with a worktree execution cutoff, the sweep skips tasks created before the cutoff, but the report still counts them. +- `ok`: counts by clause, for example `live_run`, `pending:queued_wake`, `waiting:question`. +- `violations` and `violationCount`: counts by reason (`no_pending_path`, `retry_offered`, `recovery_block`, `unknown_block`, `run_owner_gone`, `native_run_stalled`). `native_run_stalled` is a native run still marked running whose restart-recovery claim nobody renewed for 5 minutes; native recovery ends it, and it stays counted until then. +- `agentsWithStalledSlots`: up to 20 agents whose run slots are held by such stalled runs, with the number of stalled runs and of queued runs waiting behind them. +- `heldByRecoveryOwner`: violations that are blocked under an active recovery hold, which the hold's owner continues. +- `staleBlocked`: tasks blocked by recovery more than 7 days ago. They are reported, not continued. +- `settling`: tasks whose last run ended moments ago and are checked again shortly. +- `sample`: up to 20 violations with the task identifier, status and reason. + ## Org Chart ``` diff --git a/server/src/__tests__/agent-live-run-routes.test.ts b/server/src/__tests__/agent-live-run-routes.test.ts index bf511eb184f..69e704a1099 100644 --- a/server/src/__tests__/agent-live-run-routes.test.ts +++ b/server/src/__tests__/agent-live-run-routes.test.ts @@ -20,6 +20,7 @@ const mockHeartbeatService = vi.hoisted(() => ({ readLog: vi.fn(), wakeup: vi.fn(), getRun: vi.fn(), + reportAssignedWorkInvariant: vi.fn(), })); const mockIssueService = vi.hoisted(() => ({ @@ -774,6 +775,23 @@ describe("agent live run routes", () => { expect(mockWorkspaceOperationService.listForRun).toHaveBeenCalledWith("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "workspace-one"); }); + it("lets a board operator read the assigned-work invariant for their company only", async () => { + mockHeartbeatService.reportAssignedWorkInvariant.mockResolvedValue({ checked: 3, violationCount: 1, violations: { no_pending_path: 1 } }); + const board = await request(await createApp()).get("/api/companies/company-1/assigned-work-invariant"); + expect(board.status).toBe(200); + expect(board.body).toMatchObject({ checked: 3, violationCount: 1 }); + expect(mockHeartbeatService.reportAssignedWorkInvariant).toHaveBeenCalledWith("company-1", { readCondition: expect.anything() }); + + mockHeartbeatService.reportAssignedWorkInvariant.mockClear(); + const member = { type: "board", userId: "member-1", companyIds: ["company-1"], source: "session", isInstanceAdmin: false }; + const otherCompany = await request(await createApp({}, member)).get("/api/companies/company-2/assigned-work-invariant"); + expect(otherCompany.status).toBe(403); + const agent = await request(await createApp({}, { type: "agent", agentId: routeAgentId, companyId: "company-1" })) + .get("/api/companies/company-1/assigned-work-invariant"); + expect(agent.status).toBe(403); + expect(mockHeartbeatService.reportAssignedWorkInvariant).not.toHaveBeenCalled(); + }); + it("caps company live run polling by default", async () => { const rows = Array.from({ length: 75 }, (_, index) => ({ id: `run-${index}`, diff --git a/server/src/__tests__/assigned-work-invariant.test.ts b/server/src/__tests__/assigned-work-invariant.test.ts new file mode 100644 index 00000000000..1f1d296eaed --- /dev/null +++ b/server/src/__tests__/assigned-work-invariant.test.ts @@ -0,0 +1,686 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql as sqlTag } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + activityLog, agents, agentTaskSessions, agentWakeupRequests, approvals, companies, createDb, heartbeatRuns, issueApprovals, + issueRecoveryActions, issueRelations, issues, issueThreadInteractions, nativeRunFinalizations, +} from "@paperclipai/db"; +import { recoveryService } from "../services/recovery/service.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { assignedWorkSweep, ASSIGNED_WORK_CONTINUED_ACTION, ASSIGNED_WORK_REPAIRS_PER_MINUTE } from "../services/recovery/assigned-work-sweep.js"; +import { CONTINUATION_CAP } from "../services/recovery/assigned-work-invariant.js"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; + +const describePostgres = recoveryPostgresSupport.supported ? describe : describe.skip; +if (!recoveryPostgresSupport.supported) console.warn(`Skipping assigned-work invariant tests: ${recoveryPostgresSupport.reason}`); + +const HOUR = 60 * 60_000; + +/** + * The assigned-work invariant on real rows. Each stranded shape below was + * measured on staging on 2026-11-12; the existing stranded sweep leaves each + * of them with nothing pending, and the invariant sweep gives each exactly one + * ordinary continuation wake. + */ +describePostgres("assigned-work invariant sweep", () => { + let temporary: Awaited>; + let db: ReturnType; + + beforeAll(async () => { + temporary = await startRecoveryPostgres("assigned-work-invariant-"); + db = createDb(temporary.connectionString); + }, 240_000); + + afterAll(async () => { + await db?.$client.end({ timeout: 0 }); + await temporary?.cleanup(); + }); + + /** A wake queue stand-in that records each wake as a durable pending row. */ + function wakeQueue() { + const calls: Array<{ agentId: string; issueId: string; source: unknown }> = []; + const enqueueWakeup = vi.fn(async (agentId: string, opts?: { payload?: Record | null; contextSnapshot?: Record; reason?: string | null }) => { + const issueId = String(opts?.payload?.issueId); + const [issue] = await db.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, issueId)); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: issue.companyId, agentId, source: "automation", + triggerDetail: "system", reason: opts?.reason ?? null, payload: opts?.payload ?? null, status: "queued" }).returning(); + calls.push({ agentId, issueId, source: opts?.contextSnapshot?.source }); + return { id: wake.id } as never; + }); + return { calls, enqueueWakeup }; + } + + async function seedCompany() { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Invariant fixture", issuePrefix: `I${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Builder", adapterType: "codex_local", status: "idle", adapterConfig: {} }); + let number = 0; + async function issue(values: Partial = {}) { + const id = randomUUID(); + number += 1; + await db.insert(issues).values({ id, companyId, title: `Task ${number}`, status: "in_progress", assigneeAgentId: agentId, + issueNumber: number, identifier: `I${companyId.slice(0, 7)}-${number}`, updatedAt: new Date(Date.now() - 3 * HOUR), + createdAt: new Date(Date.now() - 5 * HOUR), ...values }); + return id; + } + async function run(issueId: string, values: Partial = {}) { + const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "on_demand", status: "interrupted", + errorCode: "server_shutdown_interrupted", startedAt: new Date(Date.now() - 4 * HOUR), finishedAt: new Date(Date.now() - 3 * HOUR), + livenessState: "needs_followup", contextSnapshot: { issueId, taskId: issueId }, createdAt: new Date(Date.now() - 4 * HOUR), + ...values }).returning(); + return row; + } + return { companyId, agentId, issue, run }; + } + + it("a deploy-interrupted in_progress task with nothing pending gets exactly one continuation; the stranded sweep leaves it", async () => { + const f = await seedCompany(); + const issueId = await f.issue(); + // The staging run: a person's Retry, stopped by a deploy's shutdown during + // setup, certified as no provider work, its hold already resolved. + await f.run(issueId, { resultJson: { stopReason: "interrupted", workspaceRestoreFailure: "restore_failed", + executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + contextSnapshot: { issueId, taskId: issueId, explicitUserContinuation: { previousRunId: randomUUID() } } }); + const queue = wakeQueue(); + // Production wiring: the bounded retry scheduler declines (not_scheduled). + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + + await recovery.reconcileStrandedAssignedIssues({ companyId: f.companyId, issueId, skipRelatedRecovery: true }); + expect(queue.calls).toHaveLength(0); + const [stillStranded] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(stillStranded.status).toBe("in_progress"); + + const check = await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId }); + // What staging shows for this shape: a Retry button that only the system can act on. + expect(check).toMatchObject({ outcome: "repaired", verdict: { kind: "violation", reason: "retry_offered" } }); + expect(queue.calls).toEqual([{ agentId: f.agentId, issueId, source: "issue.assigned_work_invariant" }]); + const [activity] = await db.select().from(activityLog).where(and(eq(activityLog.entityId, issueId), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION))); + expect(activity.details).toMatchObject({ message: expect.stringContaining("Continued automatically") }); + expect(String(activity.details?.message)).not.toMatch(/_|[0-9a-f]{8}-/); + + // The next sweep sees the pending wake and leaves the task alone. + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })) + .toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent", intent: "queued_wake" } }); + expect(queue.calls).toHaveLength(1); + }, 60_000); + + it("an in_progress task whose native run failed needing reconciliation (silently skipped today) gets one continuation", async () => { + const f = await seedCompany(); + const issueId = await f.issue(); + await f.run(issueId, { runtimeMode: "native", status: "failed", errorCode: "native_continuation_requires_reconciliation", + livenessState: "failed" }); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + await recovery.reconcileStrandedAssignedIssues({ companyId: f.companyId, issueId, skipRelatedRecovery: true }); + expect(queue.calls).toHaveLength(0); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })).toMatchObject({ outcome: "repaired" }); + expect(queue.calls).toHaveLength(1); + }, 60_000); + + it("a task blocked after a legacy run's retries ran out, with no hold left, is restored and continued once", async () => { + // The shape the fresh-continuation fallback does not cover: retries spent + // on a restart-caused failure (controller lease lost), block written by + // the stranded sweep, and the hold already resolved. + const f = await seedCompany(); + const blockedAt = new Date(Date.now() - 2 * HOUR); + const issueId = await f.issue({ status: "blocked", blockedTransitionAt: blockedAt }); + const run = await f.run(issueId, { status: "failed", errorCode: "controller_lease_lost", livenessState: "failed" }); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: issueId, kind: "stranded_assigned_issue", + status: "resolved", outcome: "blocked", ownerType: "board", cause: "stranded_assigned_issue", + fingerprint: `source_scoped_recovery:${f.companyId}:${issueId}:stranded_assigned_issue`, + evidence: { previousStatus: "in_progress", latestRunId: run.id, latestRunErrorCode: "controller_lease_lost" }, + nextAction: "Inspect the retry history.", attemptCount: 3, resolvedAt: blockedAt }); + await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "system", action: "issue.updated", + entityType: "issue", entityId: issueId, details: { status: "blocked", previousStatus: "in_progress", + source: "recovery.reconcile_stranded_assigned_issue" }, createdAt: blockedAt }); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup }); + + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id)).toContain(issueId); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })) + .toMatchObject({ outcome: "repaired", verdict: { reason: "recovery_block" } }); + expect(queue.calls).toHaveLength(1); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status).toBe("in_progress"); + }, 60_000); + + it("leaves a task blocked under an active recovery hold to the hold's owner and reports it", async () => { + const f = await seedCompany(); + const issueId = await f.issue({ status: "blocked", blockedTransitionAt: new Date(Date.now() - 2 * HOUR) }); + const run = await f.run(issueId, { status: "failed", errorCode: "process_lost", livenessState: "failed" }); + // Old-shaped row: a board-owned stranded action written by an older release. + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: issueId, kind: "stranded_assigned_issue", + status: "active", ownerType: "board", cause: "stranded_assigned_issue", fingerprint: `source_scoped_recovery:${f.companyId}:${issueId}:stranded_assigned_issue`, + evidence: { previousStatus: "in_progress", latestRunId: run.id, latestRunErrorCode: "process_lost" }, + nextAction: "Board operator: inspect the retry history, then explicitly retry the original owner, reassign, or intentionally resolve the task.", + attemptCount: 1 }); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup }); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })) + .toMatchObject({ outcome: "left_to_hold_owner", verdict: { kind: "violation", reason: "recovery_block" } }); + expect(queue.calls).toHaveLength(0); + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id)).not.toContain(issueId); + expect(await recovery.assignedWork.report(f.companyId)).toMatchObject({ violationCount: 1, heldByRecoveryOwner: 1 }); + }, 60_000); + + it("Retry-started tasks the scheduler refused to revalidate get one continuation, even with a question asked before the Retry", async () => { + // Nine staging tasks: a person's Retry, stopped by a deploy during setup, + // certified as no provider work; the scheduler then refused with + // continuation_user_authorization_missing on every sweep. One of them + // also had a question the agent asked before the person pressed Retry. + const f = await seedCompany(); + const plain = await f.issue(); + const withQuestion = await f.issue(); + for (const issueId of [plain, withQuestion]) { + await f.run(issueId, { resultJson: { stopReason: "interrupted", executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + contextSnapshot: { issueId, taskId: issueId, explicitUserContinuation: { previousRunId: randomUUID() } }, + createdAt: new Date(Date.now() - 4 * HOUR) }); + } + await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId: withQuestion, kind: "ask_user_questions", + status: "pending", continuationPolicy: "wake_assignee", payload: {}, createdAt: new Date(Date.now() - 12 * HOUR) } as never); + const queue = wakeQueue(); + // On staging the refusal also left the sweep's escalation without effect + // (its stale-snapshot guard): the tasks stayed in_progress with nothing + // pending. Model that by changing the task under the sweep's snapshot. + const scheduleRecoveryRetry = vi.fn(async (runId: string, outcome?: { continuationRefused?: boolean }) => { + if (outcome) outcome.continuationRefused = true; + const [run] = await db.select({ context: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + await db.update(issues).set({ updatedAt: new Date(Date.now() - 2 * HOUR) }).where(eq(issues.id, String(run.context?.issueId))); + return null; + }); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry }); + await recovery.reconcileStrandedAssignedIssues({ companyId: f.companyId, skipRelatedRecovery: true }); + expect(queue.calls).toHaveLength(0); + + for (const issueId of [plain, withQuestion]) { + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status).toBe("in_progress"); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })) + .toMatchObject({ outcome: "repaired", verdict: { reason: "retry_offered" } }); + } + expect(queue.calls.map(call => call.issueId).sort()).toEqual([plain, withQuestion].sort()); + }, 60_000); + + it("an unknown block reason is repaired, not treated as a reason to wait", async () => { + const f = await seedCompany(); + const issueId = await f.issue({ status: "blocked", blockedTransitionAt: new Date(Date.now() - HOUR) }); + await f.run(issueId, { status: "failed", errorCode: "something_new", livenessState: "failed" }); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup }); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })) + .toMatchObject({ outcome: "repaired", verdict: { reason: "unknown_block" } }); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status).toBe("todo"); + expect(queue.calls).toHaveLength(1); + }, 60_000); + + it("two servers sweeping the same violation at once create one wake", async () => { + const f = await seedCompany(); + const issueId = await f.issue(); + await f.run(issueId); + const queue = wakeQueue(); + const serverA = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }); + const serverB = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }); + const results = await Promise.all([serverA.checkIssue({ companyId: f.companyId, issueId }), serverB.checkIssue({ companyId: f.companyId, issueId })]); + expect(queue.calls).toHaveLength(1); + expect(results.map(result => result?.outcome).sort()).toEqual(["repaired", expect.stringMatching(/changed|deferred_rate_limit/)].sort()); + expect(await db.select().from(activityLog).where(and(eq(activityLog.entityId, issueId), eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION)))) + .toHaveLength(1); + }, 60_000); + + it("leaves a run owned by another live server alone, and settles one whose owner is proven gone", async () => { + const f = await seedCompany(); + const otherBoot = randomUUID(); + const controllerProcess = { bootId: otherBoot, pid: 4242, hostHash: "host-b", processStartedAt: new Date(Date.now() - HOUR).toISOString() }; + // Deploy overlap: the other server holds the lease. + const leased = await f.issue(); + await f.run(leased, { status: "running", errorCode: null, finishedAt: null, controllerBootId: otherBoot, controllerProcess, + controllerLeaseExpiresAt: new Date(Date.now() + 60_000) }); + // Lease lapsed, but the owner proof says its server is still alive. + const lapsedAlive = await f.issue(); + await f.run(lapsedAlive, { status: "running", errorCode: null, finishedAt: null, controllerBootId: otherBoot, controllerProcess, + controllerLeaseExpiresAt: new Date(Date.now() - 60_000) }); + const queue = wakeQueue(); + const alive = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup, + proveOwner: async () => ({ verdict: "alive", reason: "controller_lease_held" }) }); + expect(await alive.checkIssue({ companyId: f.companyId, issueId: leased })).toMatchObject({ verdict: { clause: "live_run" } }); + expect(await alive.checkIssue({ companyId: f.companyId, issueId: lapsedAlive })).toMatchObject({ verdict: { clause: "live_run" } }); + expect(queue.calls).toHaveLength(0); + + const settle = vi.fn(async () => undefined); + const gone = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup, settleLostRunProcess: settle, + proveOwner: async () => ({ verdict: "gone", reason: "controller_stopped" }) }); + expect(await gone.checkIssue({ companyId: f.companyId, issueId: leased })).toMatchObject({ verdict: { clause: "live_run" } }); + const goneCheck = await gone.checkIssue({ companyId: f.companyId, issueId: lapsedAlive }); + expect(goneCheck) + .toMatchObject({ outcome: "settled_lost_run", verdict: { reason: "run_owner_gone" } }); + expect(settle).toHaveBeenCalledTimes(1); + expect(queue.calls).toHaveLength(0); + }, 60_000); + + it("leaves tasks waiting on an approval, a question, a dependency or a declared unblock action", async () => { + const f = await seedCompany(); + const approvalTask = await f.issue(); + const [approval] = await db.insert(approvals).values({ companyId: f.companyId, type: "plan", status: "pending", payload: {} } as never).returning(); + await db.insert(issueApprovals).values({ companyId: f.companyId, issueId: approvalTask, approvalId: approval.id } as never); + const questionTask = await f.issue({ status: "blocked", blockedTransitionAt: new Date(Date.now() - HOUR) }); + await f.run(questionTask, { runtimeMode: "native", status: "failed", errorCode: "native_continuation_requires_reconciliation" }); + await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId: questionTask, kind: "request_confirmation", + status: "pending", continuationPolicy: "wake_assignee", payload: {} } as never); + const dependent = await f.issue({ status: "blocked", blockedTransitionAt: new Date(Date.now() - HOUR) }); + const blocker = await f.issue({ status: "todo", assigneeAgentId: null }); + await db.insert(issueRelations).values({ companyId: f.companyId, issueId: blocker, relatedIssueId: dependent, type: "blocks" } as never); + const declared = await f.issue({ status: "blocked", blockedTransitionAt: new Date(Date.now() - HOUR), + unblockDescriptor: { owner: "board", action: "Share the deploy key for the staging project." } }); + + const queue = wakeQueue(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: approvalTask })).toMatchObject({ verdict: { reason: "approval" } }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: questionTask })).toMatchObject({ verdict: { reason: "question" } }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: dependent })).toMatchObject({ verdict: { reason: "dependency" } }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: declared })).toMatchObject({ verdict: { reason: "assignee_declared_block" } }); + expect(queue.calls).toHaveLength(0); + // The indexed pre-filter leaves all four out of the sweep entirely. + const suspects = await sweep.listSuspects({ companyId: f.companyId }, null, 100); + expect(suspects.map(row => row.id)).not.toEqual(expect.arrayContaining([approvalTask])); + for (const id of [approvalTask, questionTask, dependent, declared]) expect(suspects.map(row => row.id)).not.toContain(id); + }, 60_000); + + it("a refused or later-declined continuation stays a violation, says so, and is tried again under the cap", async () => { + const f = await seedCompany(); + const issueId = await f.issue(); + await f.run(issueId); + let admit = false; + const enqueueWakeup = vi.fn(async (agentId: string, opts?: { payload?: Record | null; reason?: string | null }) => { + if (!admit) return null; + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId, source: "automation", + triggerDetail: "system", reason: opts?.reason ?? null, payload: opts?.payload ?? null, status: "queued" }).returning(); + return { id: wake.id } as never; + }); + let clock = new Date(); + const sweep = assignedWorkSweep(db, { enqueueWakeup, now: () => clock }); + const deadlines: number[] = []; + // Admission refuses: the entry says nothing started, and a re-check is due. + expect(await sweep.checkIssue({ companyId: f.companyId, issueId, onDeferredDeadline: at => deadlines.push(at) })) + .toMatchObject({ outcome: "enqueue_refused" }); + const activity = async () => db.select().from(activityLog).where(and(eq(activityLog.entityId, issueId), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION))); + expect((await activity())[0].details).toMatchObject({ outcome: "not_started", message: expect.stringContaining("could not start") }); + expect(deadlines).toHaveLength(1); + // Too soon: spaced, not repaired and not reported as fine. + clock = new Date(clock.getTime() + 60_000); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId })).toMatchObject({ outcome: "deferred_rate_limit", verdict: { kind: "violation" } }); + // After the spacing it is admitted; the queue later declines the claim + // (for example behind the old run's pending cleanup) and the wake is skipped. + admit = true; + clock = new Date(clock.getTime() + 2 * 60_000); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId })).toMatchObject({ outcome: "repaired" }); + expect((await activity()).map(row => row.details?.outcome).sort()).toEqual(["not_started", "queued"]); + await db.update(agentWakeupRequests).set({ status: "skipped" }).where(eq(agentWakeupRequests.companyId, f.companyId)); + clock = new Date(clock.getTime() + 5 * 60_000); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId })).toMatchObject({ outcome: "repaired" }); + expect(enqueueWakeup).toHaveBeenCalledTimes(3); + }, 60_000); + + it("with the real wake queue, a hold that defers the continuation leaves a violation that is retried no faster than the cap", async () => { + // Current master without a fresh-continuation fallback: the task's last + // run left a legacy reconciliation hold, so admission defers the ordinary + // continuation wake. Nothing pending is left behind, so it stays a + // violation, is retried with spacing, and never loops faster than the cap. + const f = await seedCompany(); + const issueId = await f.issue(); + const run = await f.run(issueId, { status: "failed", errorCode: "process_lost", livenessState: "failed" }); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + status: "active", ownerType: "board", cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${run.id}`, + evidence: { runId: run.id, originalFailureCode: "process_lost" }, nextAction: "Reconcile the stopped run.", attemptCount: 1 }); + const heartbeat = heartbeatService(db); + try { + const first = await heartbeat.checkAssignedWorkInvariant({ companyId: f.companyId, issueId }); + expect(first).toMatchObject({ outcome: "enqueue_refused", verdict: { kind: "violation" } }); + const queued = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sqlTag`${heartbeatRuns.status} in ('queued', 'running', 'scheduled_retry')`)); + expect(queued).toHaveLength(0); + const pending = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, f.companyId), + sqlTag`${agentWakeupRequests.status} in ('queued', 'deferred_issue_execution', 'claimed')`)); + expect(pending).toHaveLength(0); + const [entry] = await db.select().from(activityLog).where(and(eq(activityLog.entityId, issueId), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION))); + expect(entry.details).toMatchObject({ outcome: "not_started" }); + // A second sweep right away is spaced, not another attempt. + expect(await heartbeat.checkAssignedWorkInvariant({ companyId: f.companyId, issueId })) + .toMatchObject({ outcome: "deferred_rate_limit", verdict: { kind: "violation" } }); + expect(await db.select().from(activityLog).where(and(eq(activityLog.entityId, issueId), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION)))).toHaveLength(1); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); + + it("routine touches on a stranded task (a comment, a read, a label) do not postpone its repair", async () => { + const f = await seedCompany(); + const issueId = await f.issue(); + await f.run(issueId); + await db.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, issueId)); + const queue = wakeQueue(); + expect(await assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }).checkIssue({ companyId: f.companyId, issueId })) + .toMatchObject({ outcome: "repaired" }); + expect(queue.calls).toHaveLength(1); + }, 60_000); + + it("restores a block made through the normal update route to its recorded status, and a review block back to review without a wake", async () => { + const f = await seedCompany(); + const queue = wakeQueue(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }); + const blockedAt = new Date(Date.now() - 2 * HOUR); + const cases = [["in_progress", "in_progress", 1], ["in_review", "in_review", 0]] as const; + for (const [before, expected, wakes] of cases) { + const issueId = await f.issue({ status: "blocked", blockedTransitionAt: blockedAt }); + await f.run(issueId, { status: "failed", errorCode: "process_lost", livenessState: "failed" }); + // The shape the normal issue update route writes. + await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "system", action: "issue.updated", + entityType: "issue", entityId: issueId, details: { status: "blocked", _previous: { status: before } }, createdAt: blockedAt }); + const callsBefore = queue.calls.length; + expect(await sweep.checkIssue({ companyId: f.companyId, issueId })).toMatchObject({ outcome: "repaired" }); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].status).toBe(expected); + expect(queue.calls.length - callsBefore).toBe(wakes); + } + }, 60_000); + + it("puts a lifted block back when admission refuses the continuation", async () => { + const f = await seedCompany(); + const blockedAt = new Date(Date.now() - 2 * HOUR); + const issueId = await f.issue({ status: "blocked", blockedTransitionAt: blockedAt }); + await f.run(issueId, { status: "failed", errorCode: "controller_lease_lost", livenessState: "failed" }); + const notifiedAt = new Date(Date.now() - HOUR); + const descriptor = { owner: { agentId: f.agentId }, action: "Retry the build once the cache is warm." }; + await db.update(issues).set({ unblockDescriptor: descriptor as never, blockedOwnerNotifiedAt: notifiedAt }).where(eq(issues.id, issueId)); + const sweep = assignedWorkSweep(db, { enqueueWakeup: async () => null }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId })).toMatchObject({ outcome: "enqueue_refused" }); + const [after] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(after.status).toBe("blocked"); + expect(after.blockedTransitionAt?.getTime()).toBe(blockedAt.getTime()); + expect(after.unblockDescriptor).toEqual(descriptor); + expect(after.blockedOwnerNotifiedAt?.getTime()).toBe(notifiedAt.getTime()); + }, 60_000); + + it("re-runs the predicate under the row lock: a question asked after the first read stops the repair", async () => { + const f = await seedCompany(); + const issueId = await f.issue(); + await f.run(issueId); + const queue = wakeQueue(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }); + // The first read sees a violation; the person's question lands just before + // the claim's transaction takes the row lock. + const original = db.transaction.bind(db); + const spy = vi.spyOn(db, "transaction").mockImplementationOnce((async (work: never, config?: never) => { + await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId, kind: "ask_user_questions", + status: "pending", continuationPolicy: "wake_assignee", payload: {} } as never); + return original(work, config); + }) as never); + try { + expect(await sweep.checkIssue({ companyId: f.companyId, issueId })).toMatchObject({ outcome: "changed", verdict: { kind: "violation" } }); + } finally { spy.mockRestore(); } + expect(await db.select().from(activityLog).where(and(eq(activityLog.entityId, issueId), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION)))).toHaveLength(0); + expect(queue.calls).toHaveLength(0); + }, 60_000); + + it("reports only tasks the caller may read; the sweep still evaluates private tasks", async () => { + const f = await seedCompany(); + const open = await f.issue(); + await f.run(open); + const hidden = await f.issue(); + await f.run(hidden); + const queue = wakeQueue(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }); + // A caller whose read condition excludes the private task. + const report = await sweep.report(f.companyId, { readCondition: sqlTag`${issues.id} <> ${hidden}` }); + expect(report).toMatchObject({ checked: 1, violationCount: 1 }); + expect(report.sample.map(row => row.identifier)).not.toContain((await db.select().from(issues).where(eq(issues.id, hidden)))[0].identifier); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: hidden })).toMatchObject({ outcome: "repaired" }); + }, 60_000); + + it("completes a claim on a one-connection pool: the locked re-check reads through the transaction", async () => { + const single = createDb(temporary.connectionString, { maxConnections: 1 }); + try { + const f = await seedCompany(); + const issueId = await f.issue({ status: "blocked", blockedTransitionAt: new Date(Date.now() - 2 * HOUR) }); + await f.run(issueId, { status: "failed", errorCode: "controller_lease_lost", livenessState: "failed" }); + const queue = wakeQueue(); + const sweep = assignedWorkSweep(single, { enqueueWakeup: queue.enqueueWakeup }); + const result = await Promise.race([sweep.checkIssue({ companyId: f.companyId, issueId }), + new Promise(resolve => setTimeout(() => resolve("deadlocked"), 20_000))]); + expect(result).toMatchObject({ outcome: "repaired" }); + expect(queue.calls).toHaveLength(1); + } finally { await single.$client.end({ timeout: 0 }); } + }, 60_000); + + it("says when the report is capped, and that the violation count is then a lower bound", async () => { + const f = await seedCompany(); + const issueId = await f.issue(); + await f.run(issueId); + const report = await assignedWorkSweep(db, { enqueueWakeup: wakeQueue().enqueueWakeup }).report(f.companyId); + expect(report).toMatchObject({ checked: 1, openTasks: 1, truncated: false, violationCountIsLowerBound: false, violationCount: 1 }); + }, 60_000); + + it("a running run whose owner is proven gone is settled first, then the task gets its continuation within the bound", async () => { + const f = await seedCompany(); + const issueId = await f.issue(); + const otherBoot = randomUUID(); + const run = await f.run(issueId, { status: "running", errorCode: null, finishedAt: null, controllerBootId: otherBoot, + controllerProcess: { bootId: otherBoot, pid: 4242, hostHash: "host-b", processStartedAt: new Date(Date.now() - HOUR).toISOString() }, + controllerLeaseExpiresAt: new Date(Date.now() - 60_000) }); + const queue = wakeQueue(); + // Heartbeat's lost-process settle path ends the run, as it does in production. + const settle = vi.fn(async ({ runId }: { runId: string }) => { + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "process_lost", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + }); + let clock = new Date(); + const started = clock.getTime(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup, settleLostRunProcess: settle, + proveOwner: async () => ({ verdict: "gone", reason: "controller_stopped" }), now: () => clock }); + const deadlines: number[] = []; + const pass = () => sweep.checkIssue({ companyId: f.companyId, issueId, onDeferredDeadline: at => deadlines.push(at) }); + // Pass 1: the owner is proven gone outside any lock; the run goes to the settle path, no claim is tried. + expect(await pass()).toMatchObject({ outcome: "settled_lost_run", verdict: { reason: "run_owner_gone", runId: run.id } }); + expect(settle).toHaveBeenCalledTimes(1); + expect(queue.calls).toHaveLength(0); + // Pass 2 at its deadline: the run has ended and the task is repaired. + clock = new Date(deadlines.at(-1)! + 1_000); + expect(await pass()).toMatchObject({ outcome: "repaired" }); + expect(queue.calls).toHaveLength(1); + expect(clock.getTime() - started).toBeLessThan(5 * 60_000); + }, 60_000); + + it("does not count a native run left running in awaiting evidence with no renewal as live, and reports it with its agent's held slot", async () => { + // The graceful hand-off repro: the new server's startup recovery parked + // the run in awaiting evidence; nothing renews or settles it. + const f = await seedCompany(); + const stuck = await f.issue(); + const stuckRun = await f.run(stuck, { runtimeMode: "native", status: "running", errorCode: null, finishedAt: null, nativeIssueId: stuck }); + const old = new Date(Date.now() - 10 * 60_000); + await db.insert(nativeRunFinalizations).values({ runId: stuckRun.id, companyId: f.companyId, issueId: stuck, phase: "observed", + recoveryState: "awaiting_evidence", leaseExpiresAt: old, updatedAt: old }); + // A recent claim is still a live path. + const fresh = await f.issue(); + const freshRun = await f.run(fresh, { runtimeMode: "native", status: "running", errorCode: null, finishedAt: null, nativeIssueId: fresh }); + await db.insert(nativeRunFinalizations).values({ runId: freshRun.id, companyId: f.companyId, issueId: fresh, phase: "observed", + recoveryState: "awaiting_evidence", leaseExpiresAt: new Date(Date.now() + 60_000) }); + // Queued work for the same agent that cannot dispatch behind the held slot. + const waiting = await f.issue({ status: "todo" }); + await f.run(waiting, { status: "queued", errorCode: null, finishedAt: null, startedAt: null }); + const queue = wakeQueue(); + const settle = vi.fn(async () => undefined); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup, settleLostRunProcess: settle }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: stuck })) + .toMatchObject({ outcome: "left_to_native_recovery", verdict: { kind: "violation", reason: "native_run_stalled", runId: stuckRun.id } }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: fresh })).toMatchObject({ verdict: { clause: "live_run" } }); + expect(queue.calls).toHaveLength(0); + expect(settle).not.toHaveBeenCalled(); + const report = await sweep.report(f.companyId); + expect(report.violations).toMatchObject({ native_run_stalled: 1 }); + expect(report.agentsWithStalledSlots).toEqual([{ agentName: "Builder", stalledRuns: 1, queuedRuns: 1 }]); + // A caller who cannot read those tasks sees none of their runs counted. + const hiddenIds = [stuck, waiting]; + const filtered = await sweep.report(f.companyId, { readCondition: sqlTag`${issues.id} not in (${sqlTag.join(hiddenIds.map(id => sqlTag`${id}::uuid`), sqlTag`, `)})` }); + expect(filtered.violations.native_run_stalled ?? 0).toBe(0); + expect(filtered.agentsWithStalledSlots).toEqual([]); + }, 60_000); + + it("stops scheduling a task once it leaves eligibility", async () => { + const f = await seedCompany(); + const done = await f.issue(); + await f.run(done); + for (let index = 0; index < CONTINUATION_CAP; index++) { + await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "system", action: ASSIGNED_WORK_CONTINUED_ACTION, + entityType: "issue", entityId: done, details: {}, createdAt: new Date(Date.now() - (index + 1) * 8 * 60_000) }); + } + const queue = wakeQueue(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }); + const deadlines: number[] = []; + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: done, onDeferredDeadline: at => deadlines.push(at) })) + .toMatchObject({ outcome: "deferred_cap" }); + expect(deadlines).toHaveLength(1); + for (const change of [{ status: "done" }, { status: "cancelled" }, { assigneeAgentId: null, assigneeUserId: "person-1" }] as const) { + await db.update(issues).set({ status: "in_progress", assigneeAgentId: f.agentId, assigneeUserId: null, ...change }).where(eq(issues.id, done)); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: done, onDeferredDeadline: at => deadlines.push(at) })).toBeNull(); + expect((await sweep.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id)).not.toContain(done); + } + expect(deadlines).toHaveLength(1); + expect(queue.calls).toHaveLength(0); + }, 60_000); + + it("treats a stopped or blocked session goal as the person's move and only a running goal as pending", async () => { + const f = await seedCompany(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: wakeQueue().enqueueWakeup }); + const cases = [ + { goalStatus: "active", goalDesiredState: "active", expected: { clause: "pending_intent", intent: "session_goal" } }, + { goalStatus: "blocked", goalDesiredState: "paused", expected: { clause: "waiting", reason: "user_stop" } }, + { goalStatus: "active", goalDesiredState: "paused", expected: { clause: "waiting", reason: "user_stop" } }, + ]; + for (const entry of cases) { + const issueId = await f.issue(); + await f.run(issueId); + await db.insert(agentTaskSessions).values({ companyId: f.companyId, agentId: f.agentId, adapterType: "codex_local", taskKey: issueId, + goalStatus: entry.goalStatus, goalDesiredState: entry.goalDesiredState } as never); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId })).toMatchObject({ outcome: "ok", verdict: entry.expected }); + } + }, 60_000); + + it("leaves a task whose run ended seconds ago, and asks to look again after the grace", async () => { + const f = await seedCompany(); + const issueId = await f.issue(); + const finishedAt = new Date(Date.now() - 2_000); + await f.run(issueId, { status: "failed", errorCode: "process_lost", finishedAt }); + const queue = wakeQueue(); + const deadlines: number[] = []; + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId, onDeferredDeadline: at => deadlines.push(at) })) + .toMatchObject({ outcome: "settling" }); + expect(queue.calls).toHaveLength(0); + expect(deadlines).toEqual([finishedAt.getTime() + 90_000]); + }, 60_000); + + it("honours the per-task cap and the per-process repair budget; never charges the failure allowance", async () => { + const f = await seedCompany(); + const capped = await f.issue(); + await f.run(capped); + for (let index = 0; index < CONTINUATION_CAP; index++) { + await db.insert(activityLog).values({ companyId: f.companyId, actorType: "system", actorId: "system", action: ASSIGNED_WORK_CONTINUED_ACTION, + entityType: "issue", entityId: capped, details: {}, createdAt: new Date(Date.now() - (index + 1) * 8 * 60_000) }); + } + const queue = wakeQueue(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }); + const deadlines: number[] = []; + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: capped, onDeferredDeadline: at => deadlines.push(at) })) + .toMatchObject({ outcome: "deferred_cap" }); + expect(queue.calls).toHaveLength(0); + expect(deadlines[0]).toBeGreaterThan(Date.now()); + // The task stays as it was: never blocked for a person. + expect((await db.select().from(issues).where(eq(issues.id, capped)))[0].status).toBe("in_progress"); + + const many = []; + for (let index = 0; index < ASSIGNED_WORK_REPAIRS_PER_MINUTE + 2; index++) { + const id = await f.issue(); + await f.run(id); + many.push(id); + } + const outcomes = []; + for (const id of many) outcomes.push((await sweep.checkIssue({ companyId: f.companyId, issueId: id }))?.outcome); + expect(outcomes.filter(outcome => outcome === "repaired")).toHaveLength(ASSIGNED_WORK_REPAIRS_PER_MINUTE); + expect(outcomes.filter(outcome => outcome === "deferred_rate_limit")).toHaveLength(2); + // Plain wakes: no retry-of link, so no failure-budget accounting. + const wake = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId)); + expect(wake.every(row => !("retryOfRunId" in (row.payload ?? {})))).toBe(true); + }, 120_000); + + it("evaluates old-shaped rows: native awaiting reattach, a running run saved without its controller process, and stale blocks", async () => { + const f = await seedCompany(); + const reattach = await f.issue(); + const native = await f.run(reattach, { runtimeMode: "native", status: "failed", errorCode: "native_session_interrupted", + nativeIssueId: reattach }); + await db.insert(nativeRunFinalizations).values({ runId: native.id, companyId: f.companyId, issueId: reattach, phase: "observed", + recoveryState: "awaiting_runner_reattach" }); + const retryable = await f.issue(); + const failedNative = await f.run(retryable, { runtimeMode: "native", status: "failed", errorCode: "saved_native_failure", nativeIssueId: retryable }); + await db.insert(nativeRunFinalizations).values({ runId: failedNative.id, companyId: f.companyId, issueId: retryable, + phase: "retryable_failure", nextAttemptAt: new Date(Date.now() - 1_000) }); + const unstamped = await f.issue(); + await f.run(unstamped, { status: "running", errorCode: null, finishedAt: null, controllerBootId: randomUUID(), controllerProcess: null, + controllerLeaseExpiresAt: new Date(Date.now() - HOUR) }); + const stale = await f.issue({ status: "blocked", blockedTransitionAt: new Date(Date.now() - 8 * 24 * HOUR) }); + await f.run(stale, { status: "failed", errorCode: "process_lost", finishedAt: new Date(Date.now() - 8 * 24 * HOUR) }); + const queue = wakeQueue(); + const settle = vi.fn(async () => undefined); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup, settleLostRunProcess: settle, + proveOwner: async () => ({ verdict: "gone", reason: "pid_dead" }) }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: reattach })).toMatchObject({ verdict: { intent: "native_recovery" } }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: retryable })).toMatchObject({ verdict: { intent: "native_recovery" } }); + // No recorded controller process: the owner is unprovable, so it stays live. + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: unstamped })).toMatchObject({ verdict: { clause: "live_run" } }); + expect(await sweep.checkIssue({ companyId: f.companyId, issueId: stale })).toMatchObject({ outcome: "stale_reported" }); + expect(queue.calls).toHaveLength(0); + expect(settle).not.toHaveBeenCalled(); + expect((await sweep.listSuspects({ companyId: f.companyId }, null, 100)).map(row => row.id)).not.toContain(stale); + const report = await sweep.report(f.companyId); + expect(report).toMatchObject({ staleBlocked: 1, violationCount: 0 }); + }, 60_000); + + it("stays cheap at staging size: 800 open tasks and a week of runs at 2,000 a day", async () => { + const f = await seedCompany(); + const issueRows = Array.from({ length: 800 }, (_, index) => ({ id: randomUUID(), companyId: f.companyId, title: `Load ${index}`, + status: index % 3 === 0 ? "blocked" : index % 3 === 1 ? "in_progress" : "todo", assigneeAgentId: f.agentId, + issueNumber: 10_000 + index, identifier: `L${f.companyId.slice(0, 6)}-${index}`, + blockedTransitionAt: index % 3 === 0 ? new Date(Date.now() - (index % 10) * 24 * HOUR) : null, + updatedAt: new Date(Date.now() - index * 60_000) })); + for (let index = 0; index < issueRows.length; index += 200) await db.insert(issues).values(issueRows.slice(index, index + 200)); + const runs = Array.from({ length: 14_000 }, (_, index) => { + const issueId = issueRows[index % issueRows.length].id; + const at = new Date(Date.now() - (index / 14_000) * 7 * 24 * HOUR); + return { companyId: f.companyId, agentId: f.agentId, invocationSource: "automation", status: index < 40 ? "running" : "succeeded", + startedAt: at, finishedAt: index < 40 ? null : at, createdAt: at, contextSnapshot: { issueId, taskId: issueId }, + controllerLeaseExpiresAt: index < 40 ? new Date(Date.now() + 60_000) : null, controllerBootId: index < 40 ? randomUUID() : null }; + }); + for (let index = 0; index < runs.length; index += 1_000) await db.insert(heartbeatRuns).values(runs.slice(index, index + 1_000)); + await db.execute(sqlTag`analyze`); + const queue = wakeQueue(); + const sweep = assignedWorkSweep(db, { enqueueWakeup: queue.enqueueWakeup }); + + const time = async (work: () => Promise) => { const started = performance.now(); const value = await work(); + return { value, ms: Math.round(performance.now() - started) }; }; + await sweep.listSuspects({ companyId: f.companyId }, null, 100); + const suspects = await time(() => sweep.listSuspects({ companyId: f.companyId }, null, 100)); + const one = await time(() => sweep.checkIssue({ companyId: f.companyId, issueId: suspects.value[0].id })); + const report = await time(() => sweep.report(f.companyId)); + const plan = await db.execute<{ "QUERY PLAN": string }>(sqlTag`explain (analyze, buffers) select count(*) from issues i + where i.company_id = ${f.companyId} and i.status in ('todo','in_progress','blocked') and i.assignee_agent_id is not null + and not exists (select 1 from heartbeat_runs r where r.company_id = i.company_id and (r.context_snapshot->>'issueId') = i.id::text + and r.status in ('queued','running','scheduled_retry'))`); + const planText = (Array.isArray(plan) ? plan : (plan as { rows: Array<{ "QUERY PLAN": string }> }).rows).map(row => row["QUERY PLAN"]).join("\n"); + console.info(JSON.stringify({ assignedWorkQueryCost: { suspectPageMs: suspects.ms, suspects: suspects.value.length, + checkOneIssueMs: one.ms, fullReportMs: report.ms, reportChecked: report.value.checked } })); + console.info(planText); + expect(planText).not.toMatch(/Seq Scan on heartbeat_runs/); + expect(suspects.ms).toBeLessThan(1_000); + expect(one.ms).toBeLessThan(1_000); + }, 240_000); +}); + diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index 4de4e064c54..9f3bdfba4c3 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -7174,6 +7174,17 @@ export function agentRoutes( res.json(traces); }); + // Operator read of the assigned-work invariant: how many open assigned tasks + // have nothing running, nothing pending and no one else's move. Read-only; + // the sweep repairs violations by itself. Expect 0 violations after a deploy. + router.get("/companies/:companyId/assigned-work-invariant", async (req, res) => { + const companyId = req.params.companyId as string; + assertCompanyAccess(req, companyId); + assertBoard(req); + // Only tasks this caller may read count, so private tasks stay private. + res.json(await heartbeat.reportAssignedWorkInvariant(companyId, { readCondition: await issueReadSqlCondition(db, req.actor) })); + }); + router.get("/companies/:companyId/live-runs", async (req, res) => { const companyId = req.params.companyId as string; assertCompanyAccess(req, companyId); diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 5a5a89497e6..3041bcb66ef 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -8006,6 +8006,15 @@ registry.registerPath({ responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden }, }); +registry.registerPath({ + method: "get", + path: "/api/companies/{companyId}/assigned-work-invariant", + tags: ["runs"], + summary: "Report open agent tasks without a live run, pending start or wait on a person (board only)", + request: { params: z.object({ companyId: z.string() }) }, + responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden }, +}); + registry.registerPath({ method: "get", path: "/api/companies/{companyId}/live-runs", diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 89b335df43f..fce4628002e 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -24,6 +24,7 @@ export const DELIVERY_QUEUES = { orphanBlockers: "orphan-blockers", activeRecoveryActions: "active-recovery-actions", strandedAssignments: "stranded-assignments", + assignedWorkInvariant: "assigned-work-invariant", resolvedDependencies: "resolved-dependencies", taskWatchdogs: "task-watchdogs", staleIssueLocks: "stale-issue-locks", diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 95cea830e6a..220bc6aa7a5 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -4607,6 +4607,11 @@ export function heartbeatService( dependencies: reconcileResolvedDependencyWakes, locks: sweepStaleIssueLocks, watchdogs: reconcileTaskWatchdogs, + assignedWork: { + listSuspects: async (scope, cursor, limit) => recovery.assignedWork.listSuspects( + { ...scope, createdAfter: await getWorktreeExecutionCutoff() }, cursor, limit), + checkIssue: recovery.assignedWork.checkIssue, + }, schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed, }, () => canRun() && readTaskDrain(new Date()) === null), registerRunDispatchWork: (coordinator: Parameters[0], canRun: () => boolean) => @@ -4636,6 +4641,8 @@ export function heartbeatService( scheduleBoundedRetry, reconcileStrandedAssignedIssues, + checkAssignedWorkInvariant: recovery.assignedWork.checkIssue, + reportAssignedWorkInvariant: recovery.assignedWork.report, recoverPendingSessionGoalActions, recoverActiveSessionGoals, diff --git a/server/src/services/issue-recovery-notifications.ts b/server/src/services/issue-recovery-notifications.ts index 7b709960831..42c095b35b7 100644 --- a/server/src/services/issue-recovery-notifications.ts +++ b/server/src/services/issue-recovery-notifications.ts @@ -8,7 +8,7 @@ export type IssueRecoveryScope = { companyId: string; issueId?: string }; export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) { const companyHint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; const issueHint = scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*"; - for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies, + for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.assignedWorkInvariant, DELIVERY_QUEUES.resolvedDependencies, DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) { await notifyDeliveryWork(transaction, queue, queue === DELIVERY_QUEUES.strandedAssignments || queue === DELIVERY_QUEUES.staleIssueLocks ? issueHint : companyHint); diff --git a/server/src/services/recovery/assigned-work-invariant.test.ts b/server/src/services/recovery/assigned-work-invariant.test.ts new file mode 100644 index 00000000000..6be75f2a9d0 --- /dev/null +++ b/server/src/services/recovery/assigned-work-invariant.test.ts @@ -0,0 +1,235 @@ +import { describe, expect, it } from "vitest"; +import { + CONTINUATION_CAP, + CONTINUATION_CAP_WINDOW_MS, + continuationActivityMessage, + continuationAllowance, + evaluateAssignedWork, + SETTLING_GRACE_MS, + SETTLING_MAX_MS, + STALE_BLOCK_MS, + VIOLATION_REASONS, + type AssignedWorkFacts, +} from "./assigned-work-invariant.js"; + +const now = new Date("2026-11-12T14:35:00Z"); +const ago = (ms: number) => new Date(now.getTime() - ms); +const AGENT = "agent-1"; + +type Overrides = Omit, "issue"> & { issue?: Partial }; +function facts(overrides: Overrides = {}): AssignedWorkFacts { + const { issue, ...rest } = overrides; + return { + issue: { + status: "in_progress", assigneeAgentId: AGENT, assigneeUserId: null, hidden: false, + blockedTransitionAt: null, updatedAt: ago(3 * 60 * 60_000), unblockOwner: null, monitorNextCheckAt: null, + isConversation: false, awaitingFirstMessage: false, ...issue, + }, + activeRuns: [], + // The common staging shape: a legacy run interrupted by a deploy's shutdown hours ago. + latestRun: { id: "run-1", status: "interrupted", errorCode: "server_shutdown_interrupted", + finishedAt: ago(3 * 60 * 60_000), settlingUntil: null, operatorStopped: false }, + pendingWakes: 0, nativeRecoveryPending: false, pendingHandoff: false, sessionGoal: null, + interruption: null, pendingInteractions: 0, pendingApprovals: 0, openBlockers: 0, openChildren: 0, + reviewPending: false, activeRecoveryAction: null, agent: { status: "idle", invokable: true }, + companyPaused: false, pauseHold: false, budgetStop: false, blockedBy: null, + ...rest, + }; +} + +describe("assigned-work invariant", () => { + it("is not applicable to closed, unassigned, person-assigned or hidden tasks", () => { + for (const issue of [{ status: "done" }, { status: "in_review" }, { assigneeAgentId: null }, + { assigneeUserId: "user-1" }, { hidden: true }]) { + expect(evaluateAssignedWork(facts({ issue }), now)).toEqual({ kind: "not_applicable" }); + } + }); + + describe("clause 1: a live run", () => { + it("accepts a queued or running run whose owner is alive or unprovable", () => { + for (const owner of ["alive", "unprovable"] as const) { + expect(evaluateAssignedWork(facts({ activeRuns: [{ id: "r", status: "running", scheduledRetryAt: null, owner }] }), now)) + .toEqual({ kind: "ok", clause: "live_run", runId: "r" }); + } + expect(evaluateAssignedWork(facts({ activeRuns: [{ id: "q", status: "queued", scheduledRetryAt: null, owner: "alive" }] }), now)) + .toMatchObject({ kind: "ok", clause: "live_run" }); + }); + + it("does not count a native run whose restart-recovery claim nobody renews", () => { + expect(evaluateAssignedWork(facts({ activeRuns: [{ id: "n", status: "running", scheduledRetryAt: null, owner: "stalled" }] }), now)) + .toEqual({ kind: "violation", reason: "native_run_stalled", runId: "n", stale: false }); + }); + + it("does not count a running run whose owner is proven gone", () => { + expect(evaluateAssignedWork(facts({ activeRuns: [{ id: "r", status: "running", scheduledRetryAt: null, owner: "gone" }] }), now)) + .toEqual({ kind: "violation", reason: "run_owner_gone", runId: "r", stale: false }); + }); + }); + + describe("clause 2: a durable pending intent", () => { + const cases: Array<[string, Overrides]> = [ + ["queued_wake", { pendingWakes: 1 }], + ["scheduled_retry", { activeRuns: [{ id: "s", status: "scheduled_retry", scheduledRetryAt: new Date(now.getTime() + 60_000), owner: "alive" }] }], + ["native_recovery", { nativeRecoveryPending: true }], + ["pending_handoff", { pendingHandoff: true }], + ["session_goal", { sessionGoal: "active" }], + ["scheduled_monitor", { issue: { monitorNextCheckAt: new Date(now.getTime() + 60_000) } }], + ["system_recovery_working", { interruption: { responsibility: "system", state: "working", userAction: null, nextRetryAt: null, canRetry: false } }], + ]; + for (const [intent, overrides] of cases) { + it(`accepts ${intent}`, () => { + expect(evaluateAssignedWork(facts(overrides), now)).toEqual({ kind: "ok", clause: "pending_intent", intent }); + }); + } + + it("does not count a monitor or retry whose time has passed", () => { + expect(evaluateAssignedWork(facts({ issue: { monitorNextCheckAt: ago(1) } }), now)).toMatchObject({ kind: "violation" }); + expect(evaluateAssignedWork(facts({ interruption: { responsibility: "system", state: "retry_scheduled", userAction: null, + nextRetryAt: ago(1).toISOString(), canRetry: true } }), now)).toMatchObject({ kind: "violation", reason: "no_pending_path" }); + }); + }); + + describe("clause 3: someone else's move", () => { + const cases: Array<[string, Overrides]> = [ + ["question", { pendingInteractions: 1 }], + ["approval", { pendingApprovals: 1 }], + ["dependency", { openBlockers: 1 }], + ["dependency", { openChildren: 2 }], + ["review", { reviewPending: true }], + ["budget", { budgetStop: true }], + ["company_paused", { companyPaused: true }], + ["agent_paused", { agent: { status: "paused", invokable: false } }], + ["agent_paused", { pauseHold: true }], + ["user_stop", { latestRun: { id: "r", status: "cancelled", errorCode: "operator_interrupted", finishedAt: ago(60 * 60_000), + settlingUntil: null, operatorStopped: true } }], + ["user_stop", { sessionGoal: "paused" }], + ["credential", { activeRecoveryAction: { cause: "configuration_incomplete", ownerType: "board", resumeAt: null } }], + ["provider_limit", { activeRecoveryAction: { cause: "native_provider_usage_limit", ownerType: "board", + resumeAt: new Date(now.getTime() + 30 * 60_000) } }], + ["conversation", { issue: { isConversation: true } }], + ["awaiting_first_message", { issue: { awaitingFirstMessage: true } }], + ["approval", { interruption: { responsibility: "user", state: "waiting_for_user", userAction: "approval", nextRetryAt: null, canRetry: false } }], + ["credential", { interruption: { responsibility: "user", state: "waiting_for_user", userAction: "reconnect", nextRetryAt: null, canRetry: false } }], + ["board_confirmation", { interruption: { responsibility: "user", state: "waiting_for_user", userAction: "confirm_owner_stop", nextRetryAt: null, canRetry: false } }], + ["user_stop", { issue: { status: "blocked", blockedTransitionAt: ago(60 * 60_000) }, blockedBy: "user" }], + ["assignee_declared_block", { issue: { status: "blocked", blockedTransitionAt: ago(60 * 60_000) }, blockedBy: "assignee" }], + ["assignee_declared_block", { issue: { status: "blocked", blockedTransitionAt: ago(60 * 60_000), unblockOwner: "board" }, blockedBy: "unknown" }], + ]; + for (const [reason, overrides] of cases) { + it(`accepts ${reason} (${Object.keys(overrides).join(", ")})`, () => { + expect(evaluateAssignedWork(facts(overrides), now)).toEqual({ kind: "ok", clause: "waiting", reason }); + }); + } + + it("does not accept a provider limit with no resume time", () => { + expect(evaluateAssignedWork(facts({ activeRecoveryAction: { cause: "provider_quota", ownerType: "board", resumeAt: null } }), now)) + .toMatchObject({ kind: "violation" }); + }); + + it("does not accept an unblock action the assignee owes itself", () => { + expect(evaluateAssignedWork(facts({ issue: { status: "blocked", blockedTransitionAt: ago(60_000 * 60), + unblockOwner: { agentId: AGENT } }, blockedBy: "unknown" }), now)).toMatchObject({ kind: "violation", reason: "unknown_block" }); + }); + }); + + describe("violations", () => { + it("an in_progress task whose deploy-interrupted run left nothing pending", () => { + expect(evaluateAssignedWork(facts(), now)).toEqual({ kind: "violation", reason: "no_pending_path", stale: false }); + }); + + it("a task offering a Retry only the system can act on", () => { + expect(evaluateAssignedWork(facts({ interruption: { responsibility: "system", state: "failed", userAction: null, + nextRetryAt: null, canRetry: true } }), now)).toEqual({ kind: "violation", reason: "retry_offered", stale: false }); + }); + + it("an unknown hold or block reason is a violation, not a reason to wait", () => { + expect(evaluateAssignedWork(facts({ activeRecoveryAction: { cause: "some_future_cause", ownerType: "board", resumeAt: null } }), now)) + .toMatchObject({ kind: "violation", reason: "no_pending_path" }); + expect(evaluateAssignedWork(facts({ issue: { status: "blocked", blockedTransitionAt: ago(60 * 60_000) }, blockedBy: "unknown" }), now)) + .toEqual({ kind: "violation", reason: "unknown_block", stale: false }); + expect(evaluateAssignedWork(facts({ issue: { status: "blocked", blockedTransitionAt: ago(60 * 60_000) }, blockedBy: "unknown", + activeRecoveryAction: { cause: "native_session_interrupted", ownerType: "board", resumeAt: null } }), now)) + .toEqual({ kind: "violation", reason: "recovery_block", stale: false }); + expect(evaluateAssignedWork(facts({ interruption: { responsibility: "user", state: "waiting_for_user", userAction: "something_new", + nextRetryAt: null, canRetry: false } }), now)).toMatchObject({ kind: "violation" }); + }); + + it("a block by automatic recovery is a violation; one older than 7 days is reported as stale", () => { + expect(evaluateAssignedWork(facts({ issue: { status: "blocked", blockedTransitionAt: ago(60 * 60_000) }, blockedBy: "system" }), now)) + .toEqual({ kind: "violation", reason: "recovery_block", stale: false }); + expect(evaluateAssignedWork(facts({ issue: { status: "blocked", blockedTransitionAt: ago(STALE_BLOCK_MS + 1) }, blockedBy: "system" }), now)) + .toEqual({ kind: "violation", reason: "recovery_block", stale: true }); + }); + + it("every violation reason has plain activity text without codes", () => { + for (const reason of VIOLATION_REASONS) { + const text = continuationActivityMessage(reason); + expect(text).not.toMatch(/_|[0-9a-f]{8}-/); + } + }); + }); + + describe("settling: a run that ended seconds ago", () => { + it("leaves a task whose latest run ended seconds ago, with a re-check at the end of the grace", () => { + const finishedAt = ago(2_000); + expect(evaluateAssignedWork(facts({ issue: { status: "in_progress" }, latestRun: { id: "r", status: "failed", errorCode: "process_lost", + finishedAt, settlingUntil: null, operatorStopped: false } }), now)) + .toEqual({ kind: "settling", recheckAt: new Date(finishedAt.getTime() + SETTLING_GRACE_MS) }); + }); + + it("gives a new assignment and a just-resolved blocker to their own owners first, but never routine touches", () => { + const createdAt = ago(5_000); + expect(evaluateAssignedWork(facts({ issue: { status: "todo", createdAt }, latestRun: null }), now)) + .toEqual({ kind: "settling", recheckAt: new Date(createdAt.getTime() + SETTLING_GRACE_MS) }); + // A comment, read marker or label change only moves updatedAt: no postponement. + expect(evaluateAssignedWork(facts({ issue: { updatedAt: ago(1_000) } }), now)).toMatchObject({ kind: "violation" }); + const resolved = ago(10_000); + expect(evaluateAssignedWork(facts({ issue: { status: "blocked", blockedTransitionAt: ago(60 * 60_000) }, blockedBy: "assignee", + lastBlockerResolvedAt: resolved }), now)).toEqual({ kind: "settling", recheckAt: new Date(resolved.getTime() + SETTLING_GRACE_MS) }); + }); + + it("never lets a held lease postpone a finished run's repair beyond the bound", () => { + expect(evaluateAssignedWork(facts({ latestRun: { id: "r", status: "failed", errorCode: "process_lost", + finishedAt: ago(SETTLING_MAX_MS + 1_000), settlingUntil: new Date(now.getTime() + 60 * 60_000), operatorStopped: false } }), now)) + .toMatchObject({ kind: "violation" }); + }); + + it("continues a run a deploy's shutdown cut within the grace of it ending", () => { + const ended = ago(SETTLING_GRACE_MS + 1); + expect(evaluateAssignedWork(facts({ latestRun: { id: "r", status: "interrupted", errorCode: "server_shutdown_interrupted", + finishedAt: ended, settlingUntil: null, operatorStopped: false } }), now)).toMatchObject({ kind: "violation" }); + }); + + it("waits for another server's controller lease that still settles the run", () => { + const lease = new Date(now.getTime() + 60_000); + expect(evaluateAssignedWork(facts({ latestRun: { id: "r", status: "failed", errorCode: "process_lost", + finishedAt: ago(2 * 60_000), settlingUntil: lease, operatorStopped: false } }), now)) + .toEqual({ kind: "settling", recheckAt: lease }); + }); + }); +}); + +describe("continuation allowance", () => { + it("allows the first continuation and spaces later ones with doubling backoff", () => { + expect(continuationAllowance([], now)).toEqual({ allowed: true }); + expect(continuationAllowance([ago(60_000)], now)).toEqual({ allowed: false, capped: false, retryAt: new Date(now.getTime() + 60_000) }); + expect(continuationAllowance([ago(3 * 60_000)], now)).toEqual({ allowed: true }); + expect(continuationAllowance([ago(3 * 60_000), ago(10 * 60_000)], now)) + .toEqual({ allowed: false, capped: false, retryAt: new Date(now.getTime() + 60_000) }); + }); + + it("caps continuations per task per hour at the server-restart lane's limit and backs off until the window frees", () => { + const recent = Array.from({ length: CONTINUATION_CAP }, (_, index) => ago((index + 1) * 9 * 60_000)); + const result = continuationAllowance(recent, now); + expect(result).toEqual({ allowed: false, capped: true, retryAt: new Date(recent.at(-1)!.getTime() + CONTINUATION_CAP_WINDOW_MS) }); + expect(CONTINUATION_CAP).toBe(6); + }); + + it("never spaces continuations more than 15 minutes apart below the cap, and continuations an hour old no longer count", () => { + // Five continuations in the hour: 2, 4, 8, 15, 15 minutes of spacing. + const five = [ago(1 * 60_000), ago(16 * 60_000), ago(24 * 60_000), ago(28 * 60_000), ago(30 * 60_000)]; + expect(continuationAllowance(five, now)).toEqual({ allowed: false, capped: false, retryAt: new Date(now.getTime() + 14 * 60_000) }); + expect(continuationAllowance([ago(16 * 60_000), ...five.slice(1)], now)).toEqual({ allowed: true }); + expect(continuationAllowance([ago(61 * 60_000), ago(70 * 60_000)], now)).toEqual({ allowed: true }); + }); +}); diff --git a/server/src/services/recovery/assigned-work-invariant.ts b/server/src/services/recovery/assigned-work-invariant.ts new file mode 100644 index 00000000000..f9d030ea85f --- /dev/null +++ b/server/src/services/recovery/assigned-work-invariant.ts @@ -0,0 +1,340 @@ +import { SERVER_RESTART_RETRY_MAX_ATTEMPTS, SERVER_RESTART_RETRY_WINDOW_MS } from "../execution-recovery-attempt.js"; + +/** + * The assigned-work invariant. Every open task assigned to an agent (`todo`, + * `in_progress`, or `blocked` where the block is not a person's to resolve) + * has, at all times, exactly one of: + * + * 1. a live run (queued or running) whose owner is not proven gone; + * 2. a durable pending intent that will start one (a queued wake, a scheduled + * retry, a monitor check, a native recovery the coordinator still owns, a + * pending hand-off); + * 3. a named, user-visible reason it is waiting that is truly someone else's + * move. The list is closed (`WAITING_REASONS`): anything not on it is not a + * reason to wait. + * + * Anything else is a violation, and the assigned-work sweep repairs it with + * one ordinary continuation wake. This module is pure: facts in, verdict out. + */ + +/** Clause 2: a durable intent that will start a run without anyone acting. */ +export const PENDING_INTENTS = [ + "queued_wake", + "scheduled_retry", + "scheduled_monitor", + "native_recovery", + "pending_handoff", + "session_goal", + "system_recovery_working", +] as const; +export type PendingIntent = (typeof PENDING_INTENTS)[number]; + +/** Clause 3: the closed list of waits that are someone else's move. */ +export const WAITING_REASONS = [ + "question", + "approval", + "dependency", + "review", + "provider_limit", + "budget", + "credential", + "agent_paused", + "company_paused", + "user_stop", + "assignee_declared_block", + "conversation", + "awaiting_first_message", + "board_confirmation", +] as const; +export type WaitingReason = (typeof WAITING_REASONS)[number]; + +export const VIOLATION_REASONS = [ + /** A run row says running but its owning server is proven gone. */ + "run_owner_gone", + /** A native run still says running, waiting for evidence or a reattach + * that no live owner will deliver: no renewal for NATIVE_STALL_MS. It holds + * its agent's slot. Reported; its ending belongs to native recovery. */ + "native_run_stalled", + /** Open, assigned, nothing running, nothing pending, no one else's move. */ + "no_pending_path", + /** The visible state offers a Retry that only the system can act on. */ + "retry_offered", + /** Blocked by an automatic recovery outcome, not by a person. */ + "recovery_block", + /** Blocked, and nothing recorded says why. */ + "unknown_block", +] as const; +export type ViolationReason = (typeof VIOLATION_REASONS)[number]; + +/** A run's owner, judged by the shared owner proof (owner-proof.ts). */ +export type RunOwnerVerdict = "alive" | "gone" | "unprovable" | "stalled"; + +/** How long a native restart-recovery claim may go without renewal before + * its run no longer counts as live. */ +export const NATIVE_STALL_MS = 5 * 60_000; + +/** Interruption presentation `userAction`s that are a person's move. */ +const PERSON_ACTIONS: Readonly> = { + approval: "approval", + reconnect: "credential", + choose_model: "credential", + project_access: "credential", + assign_agent: "agent_paused", + budget: "budget", + resume: "agent_paused", + confirm_owner_stop: "board_confirmation", + confirm_run_stopped: "board_confirmation", +}; + +/** Active recovery-action causes that name a person's move. Every other cause + * (stranded work, interrupted sessions, unknown codes) is a recovery outcome. */ +const WAITING_RECOVERY_CAUSES: Readonly> = { + configuration_incomplete: "credential", + ownership_owner_unprovable: "board_confirmation", +}; +/** Provider usage limits are a wait only while a resume time is known. */ +const PROVIDER_LIMIT_CAUSES = new Set(["provider_quota", "native_provider_usage_limit"]); + +/** How long a just-finished run may still be settling (retry scheduling, + * claim release) before its task is judged. A re-check follows at the end. */ +export const SETTLING_GRACE_MS = 90_000; +/** Upper bound on how long another server's lease can keep a finished run settling. */ +export const SETTLING_MAX_MS = 5 * 60_000; +/** Blocked tasks older than this are reported, never continued automatically. */ +export const STALE_BLOCK_MS = 7 * 24 * 60 * 60_000; +/** Automatic continuations per task, shared with the server-restart lane. */ +export const CONTINUATION_CAP = SERVER_RESTART_RETRY_MAX_ATTEMPTS; +export const CONTINUATION_CAP_WINDOW_MS = SERVER_RESTART_RETRY_WINDOW_MS; +/** Spacing between continuations of one task, doubling with each one in the window. */ +export const CONTINUATION_BASE_BACKOFF_MS = 2 * 60_000; +/** Spacing never exceeds this below the cap, so a task that keeps needing + * continuation is retried at the hourly cap's pace, never left for hours. */ +export const CONTINUATION_MAX_BACKOFF_MS = 15 * 60_000; + +export type AssignedWorkFacts = { + issue: { + status: string; + assigneeAgentId: string | null; + assigneeUserId: string | null; + hidden: boolean; + blockedTransitionAt: Date | null; + updatedAt: Date; + /** When the task was created; a task that never ran settles from it. */ + createdAt?: Date | null; + /** Owner of the agent's declared unblock action, when it declared one. */ + unblockOwner: { agentId: string } | { userId: string } | "board" | null; + monitorNextCheckAt: Date | null; + isConversation: boolean; + awaitingFirstMessage: boolean; + }; + /** Non-terminal runs (queued, running, scheduled_retry) for this task. */ + activeRuns: Array<{ id: string; status: string; scheduledRetryAt: Date | null; owner: RunOwnerVerdict }>; + latestRun: { + id: string; + status: string; + errorCode: string | null; + finishedAt: Date | null; + /** Another server still holds this run's controller lease to settle it. */ + settlingUntil: Date | null; + operatorStopped: boolean; + } | null; + pendingWakes: number; + nativeRecoveryPending: boolean; + pendingHandoff: boolean; + sessionGoal: "active" | "paused" | null; + /** The user-visible interruption projection (task-interruption-presentation.ts). */ + interruption: { + responsibility: "system" | "user"; + state: string; + userAction: string | null; + nextRetryAt: string | null; + canRetry: boolean; + } | null; + pendingInteractions: number; + pendingApprovals: number; + openBlockers: number; + /** When the task's most recent blocker was finished, if any. */ + lastBlockerResolvedAt?: Date | null; + openChildren: number; + reviewPending: boolean; + activeRecoveryAction: { cause: string; ownerType: string; resumeAt: Date | null } | null; + agent: { status: string; invokable: boolean } | null; + companyPaused: boolean; + pauseHold: boolean; + budgetStop: boolean; + /** Who moved the task to `blocked`, from its activity. */ + blockedBy: "user" | "assignee" | "system" | "unknown" | null; +}; + +export type AssignedWorkVerdict = + | { kind: "not_applicable" } + | { kind: "ok"; clause: "live_run"; runId: string } + | { kind: "ok"; clause: "pending_intent"; intent: PendingIntent } + | { kind: "ok"; clause: "waiting"; reason: WaitingReason } + | { kind: "settling"; recheckAt: Date } + | { kind: "violation"; reason: ViolationReason; runId?: string; stale: boolean }; + +const IN_SCOPE_STATUSES = new Set(["todo", "in_progress", "blocked"]); + +export function evaluateAssignedWork(facts: AssignedWorkFacts, now: Date = new Date()): AssignedWorkVerdict { + const { issue } = facts; + if (!IN_SCOPE_STATUSES.has(issue.status) || !issue.assigneeAgentId || issue.assigneeUserId || issue.hidden) { + return { kind: "not_applicable" }; + } + + // 1. A live run with an owner that is not proven gone. An unprovable owner + // fails closed: during a deploy overlap another server may still own it. + const live = facts.activeRuns.find(run => (run.status === "queued" || run.status === "running") + && run.owner !== "gone" && run.owner !== "stalled"); + if (live) return { kind: "ok", clause: "live_run", runId: live.id }; + + // A run row still says running but its server is proven gone. It holds the + // task's execution lock, so nothing pending can start until it is settled. + const ownerGone = facts.activeRuns.find(run => run.status === "running" && run.owner === "gone"); + if (ownerGone) return { kind: "violation", reason: "run_owner_gone", runId: ownerGone.id, stale: false }; + const stalled = facts.activeRuns.find(run => run.status === "running" && run.owner === "stalled"); + if (stalled) return { kind: "violation", reason: "native_run_stalled", runId: stalled.id, stale: false }; + + // 2. A durable intent that starts a run without anyone acting. + const intent = pendingIntent(facts, now); + if (intent) return { kind: "ok", clause: "pending_intent", intent }; + + // A run that ended moments ago is still settling: its own failure path may + // be scheduling the retry right now. Look again after the grace. + const settling = settlingUntil(facts, now); + if (settling) return { kind: "settling", recheckAt: settling }; + + // 3. Someone else's move, from the closed list. + const waiting = waitingReason(facts, now); + if (waiting) return { kind: "ok", clause: "waiting", reason: waiting }; + + if (issue.status === "blocked") { + const blockedAt = issue.blockedTransitionAt ?? issue.updatedAt; + const stale = now.getTime() - blockedAt.getTime() > STALE_BLOCK_MS; + // An active recovery hold that is not a person's move (checked above) + // names the block as a recovery outcome even when no activity says who. + const recovery = facts.blockedBy === "system" || facts.activeRecoveryAction !== null; + return { kind: "violation", reason: recovery ? "recovery_block" : "unknown_block", stale }; + } + if (facts.interruption?.responsibility === "system" && facts.interruption.state === "failed" && facts.interruption.canRetry) { + return { kind: "violation", reason: "retry_offered", stale: false }; + } + return { kind: "violation", reason: "no_pending_path", stale: false }; +} + +function pendingIntent(facts: AssignedWorkFacts, now: Date): PendingIntent | null { + if (facts.pendingWakes > 0) return "queued_wake"; + if (facts.activeRuns.some(run => run.status === "scheduled_retry")) return "scheduled_retry"; + if (facts.nativeRecoveryPending) return "native_recovery"; + if (facts.pendingHandoff) return "pending_handoff"; + if (facts.sessionGoal === "active") return "session_goal"; + if (facts.issue.monitorNextCheckAt && facts.issue.monitorNextCheckAt.getTime() > now.getTime()) return "scheduled_monitor"; + const interruption = facts.interruption; + if (interruption?.responsibility === "system") { + if (interruption.state === "working") return "system_recovery_working"; + if (interruption.state === "retry_scheduled" && interruption.nextRetryAt + && Date.parse(interruption.nextRetryAt) > now.getTime()) return "scheduled_retry"; + } + return null; +} + +/** The owners of three specific events act on them first: a run's failure + * path schedules its retry when the run ends, the stranded sweep dispatches a + * new assignment, the dependency wake follows a resolved blocker. Each grants + * SETTLING_GRACE_MS from its own event; routine touches (comments, reads, + * labels) never count. A held controller lease can extend the run's window, + * but never past SETTLING_MAX_MS after the run ended. */ +function settlingUntil(facts: AssignedWorkFacts, now: Date): Date | null { + const run = facts.latestRun; + const candidates: number[] = []; + if (run?.finishedAt) { + const ended = run.finishedAt.getTime(); + candidates.push(ended + SETTLING_GRACE_MS); + if (run.settlingUntil) candidates.push(Math.min(run.settlingUntil.getTime(), ended + SETTLING_MAX_MS)); + } + // A task that never ran: its assignment was just created. + if (!run && facts.issue.createdAt) candidates.push(facts.issue.createdAt.getTime() + SETTLING_GRACE_MS); + if (facts.lastBlockerResolvedAt) candidates.push(facts.lastBlockerResolvedAt.getTime() + SETTLING_GRACE_MS); + if (!candidates.length) return null; + const until = Math.max(...candidates); + return until > now.getTime() ? new Date(until) : null; +} + +function waitingReason(facts: AssignedWorkFacts, now: Date): WaitingReason | null { + const { issue, interruption } = facts; + if (interruption?.responsibility === "user" && interruption.state === "waiting_for_user" && interruption.userAction) { + const mapped = PERSON_ACTIONS[interruption.userAction]; + if (mapped) return mapped; + } + if (facts.pendingInteractions > 0) return "question"; + if (facts.pendingApprovals > 0) return "approval"; + if (facts.openBlockers > 0 || facts.openChildren > 0) return "dependency"; + if (facts.reviewPending) return "review"; + if (facts.budgetStop) return "budget"; + if (facts.companyPaused) return "company_paused"; + if (facts.pauseHold || !facts.agent || !facts.agent.invokable) return "agent_paused"; + if (facts.sessionGoal === "paused") return "user_stop"; + if (facts.latestRun?.operatorStopped) return "user_stop"; + const action = facts.activeRecoveryAction; + if (action) { + const mapped = WAITING_RECOVERY_CAUSES[action.cause]; + if (mapped) return mapped; + if (PROVIDER_LIMIT_CAUSES.has(action.cause) && action.resumeAt && action.resumeAt.getTime() > now.getTime()) { + return "provider_limit"; + } + } + if (issue.isConversation) return "conversation"; + if (issue.awaitingFirstMessage) return "awaiting_first_message"; + if (issue.status === "blocked") { + if (facts.blockedBy === "user") return "user_stop"; + const owner = issue.unblockOwner; + const declaredForSomeoneElse = owner !== null + && !(typeof owner === "object" && "agentId" in owner && owner.agentId === issue.assigneeAgentId); + if (declaredForSomeoneElse || facts.blockedBy === "assignee") return "assignee_declared_block"; + } + return null; +} + +/** Spacing and cap for automatic continuations of one task. `recent` are the + * times of its earlier automatic continuations. At most CONTINUATION_CAP per + * rolling hour (the server-restart lane's bound). Below the cap the spacing + * doubles with each continuation in the hour, 2, 4, 8 and then 15 minutes at + * most. At the cap the next one waits until the oldest leaves the hour. */ +export function continuationAllowance(recent: Date[], now: Date = new Date()): + { allowed: true } | { allowed: false; retryAt: Date; capped: boolean } { + const inWindow = recent.filter(at => at.getTime() > now.getTime() - CONTINUATION_CAP_WINDOW_MS) + .sort((a, b) => b.getTime() - a.getTime()); + if (inWindow.length >= CONTINUATION_CAP) { + const oldest = inWindow[CONTINUATION_CAP - 1]; + return { allowed: false, capped: true, retryAt: new Date(oldest.getTime() + CONTINUATION_CAP_WINDOW_MS) }; + } + if (inWindow.length > 0) { + const spacing = Math.min(CONTINUATION_MAX_BACKOFF_MS, CONTINUATION_BASE_BACKOFF_MS * 2 ** (inWindow.length - 1)); + const next = inWindow[0].getTime() + spacing; + if (next > now.getTime()) return { allowed: false, capped: false, retryAt: new Date(next) }; + } + return { allowed: true }; +} + +/** Activity text while a continuation is being admitted, and when admission refused it. */ +export const CONTINUATION_PENDING_MESSAGE = "Starting this task again automatically after an interruption."; +export const CONTINUATION_REVIEW_MESSAGE = + "Returned to review automatically after an interruption. Nothing needed you, so the review picks up where it stopped."; +export const CONTINUATION_REFUSED_MESSAGE = + "Paperclip tried to continue this task automatically, but it could not start yet. It will try again shortly."; + +/** Plain-language activity text for an admitted continuation. No codes, no ids. */ +export function continuationActivityMessage(reason: ViolationReason): string { + switch (reason) { + case "native_run_stalled": + case "run_owner_gone": + return "Continued automatically: the server running this task stopped, so Paperclip queued it again."; + case "recovery_block": + case "unknown_block": + return "Continued automatically after an interruption. Nothing needed you, so Paperclip queued the task again."; + case "retry_offered": + return "Continued automatically after an interruption. Paperclip queued the task again, so you don't need to press Retry."; + default: + return "Continued automatically after an interruption. Nothing was running or scheduled, so Paperclip queued the task again."; + } +} diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts new file mode 100644 index 00000000000..ebe6302bdc0 --- /dev/null +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -0,0 +1,708 @@ +import { and, desc, eq, gte, inArray, isNull, or, sql, type SQL, type SQLWrapper } from "drizzle-orm"; +import { + activityLog, + agentTaskSessions, + agentWakeupRequests, + agents, + approvals, + caseHandoffs, + companies, + heartbeatRuns, + issueApprovals, + issueRelations, + issueThreadInteractions, + issues, + nativeRunFinalizations, + type Db, +} from "@paperclipai/db"; +import { logger } from "../../middleware/logger.js"; +import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; +import { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js"; +import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js"; +import { unadmittedChatWakeupCondition } from "../durable-chat-wakeup.js"; +import { issueRecoveryActionService } from "../issue-recovery-actions.js"; +import { issueService } from "../issues.js"; +import { legacyControllerOwnerStamp } from "../legacy-controller-lease.js"; +import { currentProcessProbe, proveOwnerGone, type OwnerProof, type LocalOwnerStamp } from "../owner-proof.js"; +import { issueTreeControlService } from "../issue-tree-control.js"; +import { taskInterruptionPresentations } from "../task-interruption-presentation.js"; +import { CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js"; +import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js"; +import { + continuationActivityMessage, + continuationAllowance, + CONTINUATION_PENDING_MESSAGE, + CONTINUATION_REFUSED_MESSAGE, + CONTINUATION_REVIEW_MESSAGE, + evaluateAssignedWork, + CONTINUATION_CAP_WINDOW_MS, + STALE_BLOCK_MS, + NATIVE_STALL_MS, + type AssignedWorkFacts, + type AssignedWorkVerdict, + type RunOwnerVerdict, + type ViolationReason, +} from "./assigned-work-invariant.js"; + +/** Activity action recorded for every automatic continuation. It is also the + * durable claim two servers serialize on, and the per-task rate-limit ledger. */ +export const ASSIGNED_WORK_CONTINUED_ACTION = "issue.continued_automatically"; +export const ASSIGNED_WORK_CONTINUATION_SOURCE = "issue.assigned_work_invariant"; +/** Repairs per process per minute, so a deploy with many violations does not + * start hundreds of runs at once. The rest are deferred, newest first. */ +export const ASSIGNED_WORK_REPAIRS_PER_MINUTE = 10; +/** Upper bound on tasks one report reads. */ +export const ASSIGNED_WORK_REPORT_LIMIT = 2_000; + +const OPEN_STATUSES = ["todo", "in_progress", "blocked"] as const; +const PENDING_WAKE_STATUSES = ["queued", "deferred_issue_execution", "claimed"]; +const NON_TERMINAL_RUN_STATUSES = ["queued", "running", "scheduled_retry"]; +const NATIVE_PENDING_RECOVERY_STATES = ["awaiting_evidence", "awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"]; +const PROVIDER_LIMIT_CAUSES = ["provider_quota", "native_provider_usage_limit"]; +const BLOCK_ATTRIBUTION_WINDOW_MS = 10_000; + +type IssueRow = Pick; + +type RunRow = Pick & { issueId: string }; + +export type AssignedWorkDeps = { + enqueueWakeup: (agentId: string, opts: { + source: "automation" | "assignment"; triggerDetail: "system"; reason: string; + payload: Record; requestedByActorType: "system"; requestedByActorId: null; + contextSnapshot: Record; + }) => Promise<{ id: string } | null>; + /** Runs this process executes now. */ + liveRunExecutions?: Readonly<{ has(id: string): boolean }>; + /** Heartbeat's lost-process settle path for a running run whose owner is gone. */ + settleLostRunProcess?: (scope: { companyId: string; runId: string; onDeferredDeadline?: (at: number) => void }) => Promise; + /** Shared owner proof, injectable for tests. */ + proveOwner?: (stamp: LocalOwnerStamp) => Promise; + now?: () => Date; +}; + +export type AssignedWorkCheck = { + verdict: AssignedWorkVerdict; + outcome: "ok" | "settling" | "repaired" | "settled_lost_run" | "deferred_rate_limit" | "deferred_cap" | "stale_reported" | "left_to_hold_owner" | "left_to_native_recovery" | "changed" | "enqueue_refused"; + retryAt: Date | null; +}; + +const json = (value: unknown): Record => + value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + +/** Per-pass accounting, logged once per burst of checks (no timer while idle). */ +function createPassLog() { + let pass: { startedAt: number; checked: number; ok: Record; violations: Record; + repaired: number; deferred: number; stale: number } | null = null; + let timer: NodeJS.Timeout | null = null; + const flush = () => { + timer = null; + if (!pass) return; + const summary = pass; pass = null; + logger.info({ event: "assigned_work_invariant.pass", checked: summary.checked, ok: summary.ok, + violations: summary.violations, repaired: summary.repaired, deferredByRateLimit: summary.deferred, + staleBlockedReported: summary.stale, durationMs: Date.now() - summary.startedAt }, "assigned-work invariant pass"); + }; + return { + record(check: AssignedWorkCheck) { + pass ??= { startedAt: Date.now(), checked: 0, ok: {}, violations: {}, repaired: 0, deferred: 0, stale: 0 }; + pass.checked += 1; + const verdict = check.verdict; + if (verdict.kind === "ok") { + const key = verdict.clause === "live_run" ? "live_run" : verdict.clause === "pending_intent" ? `pending:${verdict.intent}` : `waiting:${verdict.reason}`; + pass.ok[key] = (pass.ok[key] ?? 0) + 1; + } else if (verdict.kind === "violation") { + pass.violations[verdict.reason] = (pass.violations[verdict.reason] ?? 0) + 1; + } + if (check.outcome === "repaired" || check.outcome === "settled_lost_run") pass.repaired += 1; + if (check.outcome === "deferred_rate_limit" || check.outcome === "deferred_cap") pass.deferred += 1; + if (check.outcome === "stale_reported") pass.stale += 1; + if (!timer) { timer = setTimeout(flush, 5_000); timer.unref?.(); } + }, + flush, + }; +} + +export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { + const now = () => deps.now?.() ?? new Date(); + const proveOwner = deps.proveOwner ?? ((stamp: LocalOwnerStamp) => proveOwnerGone(stamp, { ...currentProcessProbe(), readOnly: true })); + const passLog = createPassLog(); + const repairTimes: number[] = []; + const sweepTreeControl = issueTreeControlService(db); + + async function runOwner(run: RunRow, at: Date, proveOwners = true): Promise { + if (run.status !== "running") return "alive"; + if (deps.liveRunExecutions?.has(run.id)) return "alive"; + // Native runs are owned through the native coordinator's fenced claims, + // which the native settle watch judges. Never declare them gone here. + if (run.runtimeMode !== "legacy") return "unprovable"; + // A held lease is a live owner, including another server during a deploy overlap. + if (run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt.getTime() > at.getTime()) return "alive"; + const stamp = legacyControllerOwnerStamp(run); + if (!stamp || !proveOwners) return "unprovable"; + const proof = await proveOwner(stamp); + return proof.verdict; + } + + /** `database` is the connection to read through: the claim passes its + * transaction, so the re-check holds no second pool connection while the + * task row is locked and sees exactly what the lock protects. Owner proofs + * that reach other connections are skipped there (`proveOwners: false`): + * a lapsed-lease run is then treated as live, which only stops the repair. */ + async function loadFacts(companyId: string, issueRows: IssueRow[], + options: { database?: Db; proveOwners?: boolean } = {}): Promise> { + const database = options.database ?? db; + const proveOwners = options.proveOwners ?? true; + const treeControl = database === db ? sweepTreeControl : issueTreeControlService(database); + const result = new Map(); + if (!issueRows.length) return result; + const at = now(); + const ids = issueRows.map(issue => issue.id); + const idParam = sql`${sql.param(ids)}::text[]`; + const runIssue = sql`${heartbeatRuns.contextSnapshot}->>'issueId'`; + const runColumns = { + issueId: runIssue, id: heartbeatRuns.id, status: heartbeatRuns.status, agentId: heartbeatRuns.agentId, + runtimeMode: heartbeatRuns.runtimeMode, errorCode: heartbeatRuns.errorCode, finishedAt: heartbeatRuns.finishedAt, + scheduledRetryAt: heartbeatRuns.scheduledRetryAt, controllerBootId: heartbeatRuns.controllerBootId, + controllerLeaseExpiresAt: heartbeatRuns.controllerLeaseExpiresAt, controllerProcess: heartbeatRuns.controllerProcess, + resultJson: sql | null>`jsonb_build_object('cancelledByActorType', ${heartbeatRuns.resultJson}->'cancelledByActorType')`, + }; + const agentIds = [...new Set(issueRows.flatMap(issue => issue.assigneeAgentId ? [issue.assigneeAgentId] : []))]; + const blockedRows = issueRows.filter(issue => issue.status === "blocked"); + const notSupersededByUserContinuation = interactionNotSupersededSql(issueThreadInteractions.companyId, + sql`${issueThreadInteractions.issueId}::text`, issueThreadInteractions.createdAt); + + const [activeRuns, latestRuns, wakes, chatWakes, nativePending, handoffs, goals, interactions, approvalRows, + blockerRows, resolvedBlockerRows, childRows, actionMap, agentRows, companyRows, onboardingRows, blockActivity, interruptions] = await Promise.all([ + database.select(runColumns).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, NON_TERMINAL_RUN_STATUSES), sql`${runIssue} = any(${idParam})`)), + // The newest run per task, read through the (company, issueId, created) index. + database.execute(sql`select requested.id as "issueId", latest.* + from unnest(${idParam}) as requested(id) + cross join lateral ( + select r.id, r.status, r.agent_id as "agentId", r.runtime_mode as "runtimeMode", r.error_code as "errorCode", + r.finished_at as "finishedAt", r.scheduled_retry_at as "scheduledRetryAt", + r.controller_boot_id as "controllerBootId", r.controller_lease_expires_at as "controllerLeaseExpiresAt", + r.controller_process as "controllerProcess", + jsonb_build_object('cancelledByActorType', r.result_json->'cancelledByActorType') as "resultJson" + from heartbeat_runs r + where r.company_id = ${companyId} and (r.context_snapshot->>'issueId') = requested.id + order by r.created_at desc limit 1 + ) latest`), + database.select({ issueId: sql`${agentWakeupRequests.payload}->>'issueId'`, count: sql`count(*)::int` }) + .from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId), + inArray(agentWakeupRequests.status, PENDING_WAKE_STATUSES), + sql`${agentWakeupRequests.payload}->>'issueId' = any(${idParam})`)) + .groupBy(sql`${agentWakeupRequests.payload}->>'issueId'`), + database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, ids), + unadmittedChatWakeupCondition(issues.id, issues.companyId))), + database.select({ issueId: nativeRunFinalizations.issueId, runId: nativeRunFinalizations.runId, + recoveryState: nativeRunFinalizations.recoveryState, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt, + updatedAt: nativeRunFinalizations.updatedAt }).from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, companyId), inArray(nativeRunFinalizations.issueId, ids), + isNull(nativeRunFinalizations.resultId), + // Same native paths as the stranded sweep's active-path check: a + // restart recovery the coordinator owns, or a retryable failure it + // will finalize — unless a person's Retry already replaced the run. + or(inArray(nativeRunFinalizations.recoveryState, NATIVE_PENDING_RECOVERY_STATES), + eq(nativeRunFinalizations.phase, "retryable_failure")), + sql`coalesce(${nativeRunFinalizations.failureDetail}->>'replacementDenied', '') <> 'explicit_user_continuation'`)), + database.selectDistinct({ issueId: caseHandoffs.issueId }).from(caseHandoffs).where(and(eq(caseHandoffs.companyId, companyId), + inArray(caseHandoffs.issueId, ids), eq(caseHandoffs.destinationKind, "agent"), + inArray(caseHandoffs.state, ["awaiting_release", "pending_dispatch"]))), + database.select({ taskKey: agentTaskSessions.taskKey, agentId: agentTaskSessions.agentId, goalStatus: agentTaskSessions.goalStatus, + goalDesiredState: agentTaskSessions.goalDesiredState }) + .from(agentTaskSessions).where(and(eq(agentTaskSessions.companyId, companyId), inArray(agentTaskSessions.taskKey, ids), + sql`${agentTaskSessions.goalStatus} is not null`, sql`${agentTaskSessions.goalStatus} <> 'complete'`)), + // A pending question counts as the person's move until the person + // answers it another way: a Retry or message started after the question + // was asked (an explicit user continuation) supersedes it as a wait. + database.select({ issueId: issueThreadInteractions.issueId, count: sql`count(*)::int` }).from(issueThreadInteractions) + .where(and(eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.issueId, ids), + eq(issueThreadInteractions.status, "pending"), notSupersededByUserContinuation)) + .groupBy(issueThreadInteractions.issueId), + database.select({ issueId: issueApprovals.issueId, count: sql`count(*)::int` }).from(issueApprovals) + .innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)) + .where(and(eq(issueApprovals.companyId, companyId), inArray(issueApprovals.issueId, ids), eq(approvals.status, "pending"))) + .groupBy(issueApprovals.issueId), + database.select({ issueId: issueRelations.relatedIssueId, count: sql`count(*)::int` }).from(issueRelations) + .innerJoin(issues, and(eq(issues.id, issueRelations.issueId), eq(issues.companyId, issueRelations.companyId))) + .where(and(eq(issueRelations.companyId, companyId), inArray(issueRelations.relatedIssueId, ids), + eq(issueRelations.type, "blocks"), sql`${issues.status} not in ('done', 'cancelled')`)) + .groupBy(issueRelations.relatedIssueId), + database.select({ issueId: issueRelations.relatedIssueId, + at: sql`max(coalesce(${issues.completedAt}, ${issues.cancelledAt}))` }).from(issueRelations) + .innerJoin(issues, and(eq(issues.id, issueRelations.issueId), eq(issues.companyId, issueRelations.companyId))) + .where(and(eq(issueRelations.companyId, companyId), inArray(issueRelations.relatedIssueId, ids), + eq(issueRelations.type, "blocks"), sql`${issues.status} in ('done', 'cancelled')`)) + .groupBy(issueRelations.relatedIssueId), + database.select({ parentId: issues.parentId, count: sql`count(*)::int` }).from(issues) + .where(and(eq(issues.companyId, companyId), inArray(issues.parentId, ids), sql`${issues.status} not in ('done', 'cancelled')`, + isNull(issues.hiddenAt))).groupBy(issues.parentId), + issueRecoveryActionService(database).listActiveForIssues(companyId, ids), + agentIds.length ? database.select().from(agents).where(and(eq(agents.companyId, companyId), inArray(agents.id, agentIds))) : [], + database.select({ status: companies.status, pauseReason: companies.pauseReason }).from(companies).where(eq(companies.id, companyId)), + database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, ids), + onboardingFirstTaskAwaitingUserSql(issues.companyId, issues.id, issues.originKind))), + blockedRows.length ? database.execute<{ issueId: string; actorType: string; actorId: string; agentId: string | null; details: unknown }>(sql` + select requested.id as "issueId", latest.actor_type as "actorType", latest.actor_id as "actorId", + latest.agent_id as "agentId", latest.details + from jsonb_to_recordset(${JSON.stringify(blockedRows.map(issue => ({ id: issue.id, + blocked_at: (issue.blockedTransitionAt ?? issue.updatedAt).toISOString() })))}::jsonb) as requested(id text, blocked_at timestamptz) + cross join lateral ( + select a.actor_type, a.actor_id, a.agent_id, a.details from activity_log a + where a.company_id = ${companyId} and a.entity_type = 'issue' and a.entity_id = requested.id + and a.created_at between requested.blocked_at - make_interval(secs => ${BLOCK_ATTRIBUTION_WINDOW_MS / 1000}) + and requested.blocked_at + make_interval(secs => ${BLOCK_ATTRIBUTION_WINDOW_MS / 1000}) + and (a.details->>'status' = 'blocked' or a.details->>'toStatus' = 'blocked' + or a.details->'changes'->'status'->>'to' = 'blocked') + order by a.created_at desc limit 1 + ) latest`) : [], + taskInterruptionPresentations(database, companyId, issueRows.map(issue => ({ id: issue.id, companyId: issue.companyId, + status: issue.status, assigneeAgentId: issue.assigneeAgentId, projectId: issue.projectId }))), + ]); + + const rowsOf = (value: unknown): T[] => Array.isArray(value) ? value as T[] : ((value as { rows?: T[] })?.rows ?? []); + const agentsById = new Map(); + for (const agent of agentRows as Array) { + // An agent still being set up, or held by a lifecycle hold, takes no work + // yet; its setup or release is the next move, not a continuation. + const ready = !isAgentAwaitingSetup(agent) && (agent.lifecycleHolds ?? []).length === 0; + agentsById.set(agent.id, { ...agent, invokable: ready && (await evaluateAgentInvokabilityFromDb(database, agent)).invokable }); + } + const company = companyRows[0]; + const companyBudget = company?.pauseReason === "budget" || company?.pauseReason === "budget_exceeded"; + const counts = (rows: Array<{ issueId: string | null; count: number }>) => + new Map(rows.filter(row => row.issueId).map(row => [row.issueId!, Number(row.count)])); + const wakeCounts = counts(wakes); + const chatWakeIds = new Set(chatWakes.map(row => row.id)); + // A native restart-recovery claim that nobody renewed for NATIVE_STALL_MS + // is not a pending path, and its still-running run is not live. + const stalledRunIds = new Set(nativePending.filter(row => row.recoveryState + && !deps.liveRunExecutions?.has(row.runId) + && (!row.leaseExpiresAt || row.leaseExpiresAt.getTime() < at.getTime() - NATIVE_STALL_MS) + && row.updatedAt.getTime() < at.getTime() - NATIVE_STALL_MS).map(row => row.runId)); + const nativeIds = new Set(nativePending.filter(row => !stalledRunIds.has(row.runId)).map(row => row.issueId)); + const handoffIds = new Set(handoffs.map(row => row.issueId)); + const interactionCounts = counts(interactions); + const approvalCounts = counts(approvalRows); + const blockerCounts = counts(blockerRows); + const resolvedBlockerAt = new Map(resolvedBlockerRows.filter(row => row.at).map(row => [row.issueId, new Date(row.at!)])); + const childCounts = counts(childRows.map(row => ({ issueId: row.parentId, count: row.count }))); + const onboardingIds = new Set(onboardingRows.map(row => row.id)); + const latestByIssue = new Map(rowsOf(latestRuns).map(row => [row.issueId, row])); + const blockActivityByIssue = new Map(rowsOf<{ issueId: string; actorType: string; actorId: string; agentId: string | null; details: unknown }>(blockActivity) + .map(row => [row.issueId, row])); + + for (const issue of issueRows) { + const agent = issue.assigneeAgentId ? agentsById.get(issue.assigneeAgentId) ?? null : null; + const runs = activeRuns.filter(run => run.issueId === issue.id) as RunRow[]; + const ownedRuns = []; + for (const run of runs) { + ownedRuns.push({ id: run.id, status: run.status, scheduledRetryAt: run.scheduledRetryAt, owner: stalledRunIds.has(run.id) && run.status === "running" ? "stalled" : await runOwner(run, at, proveOwners) }); + } + const latest = latestByIssue.get(issue.id) ?? null; + const latestFinishedAt = latest?.finishedAt ? new Date(latest.finishedAt) : null; + const leaseUntil = latest?.controllerLeaseExpiresAt ? new Date(latest.controllerLeaseExpiresAt) : null; + const goal = goals.find(row => row.taskKey === issue.id && row.agentId === issue.assigneeAgentId); + const action = actionMap.get(issue.id) ?? null; + const interruption = interruptions.get(issue.id) ?? null; + const executionState = json(issue.executionState); + const blockedEntry = blockActivityByIssue.get(issue.id); + // The tree pause gate walks ancestors; read it only when nothing else + // already gives the task a path. + const needsPauseCheck = !ownedRuns.some(run => run.owner !== "gone") && !(wakeCounts.get(issue.id) ?? 0); + const pauseHold = needsPauseCheck && Boolean(await treeControl.getActivePauseHoldGate(companyId, issue.id)); + result.set(issue.id, { + issue: { + status: issue.status, assigneeAgentId: issue.assigneeAgentId, assigneeUserId: issue.assigneeUserId, + hidden: Boolean(issue.hiddenAt), blockedTransitionAt: issue.blockedTransitionAt, updatedAt: issue.updatedAt, + createdAt: issue.createdAt, + unblockOwner: (issue.unblockDescriptor?.owner ?? null) as AssignedWorkFacts["issue"]["unblockOwner"], + monitorNextCheckAt: issue.monitorNextCheckAt, + isConversation: Boolean(issue.conversationAgentId) || issue.originKind === "chat_channel", + awaitingFirstMessage: onboardingIds.has(issue.id), + }, + activeRuns: ownedRuns, + latestRun: latest ? { + id: latest.id, status: latest.status, errorCode: latest.errorCode, finishedAt: latestFinishedAt, + settlingUntil: latest.runtimeMode === "legacy" && latest.controllerBootId && leaseUntil + && !NON_TERMINAL_RUN_STATUSES.includes(latest.status) ? leaseUntil : null, + operatorStopped: latest.agentId === issue.assigneeAgentId && ( + [CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE, "operator_interrupted"].includes(latest.errorCode ?? "") + || (latest.status === "cancelled" && ["user", "board"].includes(String(json(latest.resultJson).cancelledByActorType)))), + } : null, + pendingWakes: (wakeCounts.get(issue.id) ?? 0) + (chatWakeIds.has(issue.id) ? 1 : 0), + nativeRecoveryPending: nativeIds.has(issue.id), + pendingHandoff: handoffIds.has(issue.id), + // Only a goal that is running and still wanted running starts turns by + // itself; a paused, blocked or stopped goal waits for its person. + sessionGoal: goal ? (goal.goalStatus === "active" && goal.goalDesiredState !== "paused" ? "active" : "paused") : null, + interruption: interruption ? { responsibility: interruption.responsibility, state: interruption.state, + userAction: interruption.userAction, nextRetryAt: interruption.nextRetryAt, canRetry: interruption.canRetry } : null, + pendingInteractions: interactionCounts.get(issue.id) ?? 0, + pendingApprovals: approvalCounts.get(issue.id) ?? 0, + openBlockers: blockerCounts.get(issue.id) ?? 0, + lastBlockerResolvedAt: resolvedBlockerAt.get(issue.id) ?? null, + openChildren: childCounts.get(issue.id) ?? 0, + reviewPending: executionState.status === "pending" || Boolean(executionState.reviewRequest), + activeRecoveryAction: action ? { cause: action.cause, ownerType: action.ownerType, + resumeAt: PROVIDER_LIMIT_CAUSES.includes(action.cause) && interruption?.nextRetryAt ? new Date(interruption.nextRetryAt) : null } : null, + agent: agent ? { status: agent.status, invokable: agent.invokable } : null, + companyPaused: company?.status === "paused", + pauseHold, + budgetStop: companyBudget || agent?.pauseReason === "budget" || agent?.pauseReason === "budget_exceeded", + blockedBy: issue.status !== "blocked" ? null : attributeBlock(blockedEntry, issue.assigneeAgentId), + }); + } + return result; + } + + const issueColumns = { + id: issues.id, companyId: issues.companyId, status: issues.status, assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, hiddenAt: issues.hiddenAt, blockedTransitionAt: issues.blockedTransitionAt, + updatedAt: issues.updatedAt, unblockDescriptor: issues.unblockDescriptor, monitorNextCheckAt: issues.monitorNextCheckAt, + conversationAgentId: issues.conversationAgentId, originKind: issues.originKind, projectId: issues.projectId, + statusVersion: issues.statusVersion, executionState: issues.executionState, identifier: issues.identifier, + createdAt: issues.createdAt, blockedOwnerNotifiedAt: issues.blockedOwnerNotifiedAt, + }; + + async function loadIssues(companyId: string, issueId?: string, limit = ASSIGNED_WORK_REPORT_LIMIT, + readCondition?: SQL): Promise { + return db.select(issueColumns).from(issues).where(and( + eq(issues.companyId, companyId), + readCondition, + issueId ? eq(issues.id, issueId) : undefined, + inArray(issues.status, [...OPEN_STATUSES]), + sql`${issues.assigneeAgentId} is not null`, + isNull(issues.assigneeUserId), + isNull(issues.hiddenAt), + )).orderBy(desc(issues.updatedAt)).limit(limit); + } + + /** + * The cheap, indexed pre-filter the sweep reconstructs from: open assigned + * tasks with no run that is plainly live, no pending wake and no pending + * question. Everything it lets through is judged by the full predicate. + * Chat conversations and blocks older than the stale window are left out: + * the first wait on a person's message, the second is only reported. + */ + async function listSuspects(scope: { companyId?: string; issueId?: string; createdAfter?: Date | null }, cursor: string | null, limit: number) { + const at = now(); + return db.select({ id: issues.id, companyId: issues.companyId, updatedAt: issues.updatedAt }).from(issues).where(and( + scope.companyId ? eq(issues.companyId, scope.companyId) : undefined, + scope.issueId ? eq(issues.id, scope.issueId) : undefined, + cursor ? sql`${issues.id} > ${cursor}` : undefined, + scope.createdAfter ? gte(issues.createdAt, scope.createdAfter) : undefined, + inArray(issues.status, [...OPEN_STATUSES]), + sql`${issues.assigneeAgentId} is not null`, + isNull(issues.assigneeUserId), + isNull(issues.hiddenAt), + isNull(issues.conversationAgentId), + sql`coalesce(${issues.originKind}, '') <> 'chat_channel'`, + sql`(${issues.status} <> 'blocked' or coalesce(${issues.blockedTransitionAt}, ${issues.updatedAt}) > ${new Date(at.getTime() - STALE_BLOCK_MS).toISOString()}::timestamptz)`, + sql`(${issues.monitorNextCheckAt} is null or ${issues.monitorNextCheckAt} <= ${at.toISOString()}::timestamptz)`, + // A legacy run whose controller lease lapsed may have lost its owner; + // the predicate judges it with the owner proof. Other active runs are live. + sql`not exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} + and (r.context_snapshot->>'issueId') = ${issues.id}::text + and r.status in ('queued', 'running', 'scheduled_retry') + and not (r.status = 'running' and r.runtime_mode = 'legacy' and r.controller_lease_expires_at is not null + and r.controller_lease_expires_at < ${at.toISOString()}::timestamptz))`, + sql`not exists (select 1 from agent_wakeup_requests w where w.company_id = ${issues.companyId} + and (w.payload->>'issueId') = ${issues.id}::text + and w.status in ('queued', 'deferred_issue_execution', 'claimed'))`, + sql`not exists (select 1 from issue_thread_interactions t where t.company_id = ${issues.companyId} + and t.issue_id = ${issues.id} and t.status = 'pending' + and ${interactionNotSupersededSql(sql`t.company_id`, sql`t.issue_id::text`, sql`t.created_at`)})`, + // Plain waits on someone else, decided without the full read. The + // predicate re-checks every task this lets through. + sql`not exists (select 1 from issue_relations rel inner join issues blocker + on blocker.id = rel.issue_id and blocker.company_id = rel.company_id + where rel.company_id = ${issues.companyId} and rel.related_issue_id = ${issues.id} + and rel.type = 'blocks' and blocker.status not in ('done', 'cancelled'))`, + sql`not exists (select 1 from issue_approvals ia inner join approvals ap on ap.id = ia.approval_id + where ia.company_id = ${issues.companyId} and ia.issue_id = ${issues.id} and ap.status = 'pending')`, + // A credential or board-confirmation hold waits on a person; any active + // hold on a blocked task is re-continued by the hold's owner. + sql`not exists (select 1 from issue_recovery_actions ra where ra.company_id = ${issues.companyId} + and ra.source_issue_id = ${issues.id} and ra.status in ('active', 'escalated') + and (ra.cause in ('configuration_incomplete', 'ownership_owner_unprovable') or ${issues.status} = 'blocked'))`, + // An assignee that cannot take work (paused, being set up, held) is + // its own wait; the predicate names it when anything else asks. + sql`exists (select 1 from agents a where a.id = ${issues.assigneeAgentId} and a.company_id = ${issues.companyId} + and a.status not in ('paused', 'terminated', 'pending_approval') + and a.lifecycle_state not in ('preparing', 'verifying', 'resuming') + and jsonb_array_length(a.lifecycle_holds) = 0)`, + sql`(${issues.status} <> 'blocked' or ${issues.unblockDescriptor} is null + or (${issues.unblockDescriptor}->'owner'->>'agentId') = ${issues.assigneeAgentId}::text)`, + )).orderBy(issues.id).limit(limit); + } + + /** Evaluate one task, and repair it when it violates the invariant. */ + async function checkIssue(scope: { companyId: string; issueId: string; onDeferredDeadline?: (at: number) => void }): Promise { + const [issue] = await loadIssues(scope.companyId, scope.issueId, 1); + if (!issue) return null; + const facts = (await loadFacts(scope.companyId, [issue])).get(issue.id)!; + const at = now(); + const verdict = evaluateAssignedWork(facts, at); + const check = await act(issue, facts, verdict, at); + if (check.retryAt) scope.onDeferredDeadline?.(check.retryAt.getTime()); + passLog.record(check); + return check; + } + + async function act(issue: IssueRow, facts: AssignedWorkFacts, verdict: AssignedWorkVerdict, at: Date): Promise { + if (verdict.kind === "not_applicable" || verdict.kind === "ok") return { verdict, outcome: "ok", retryAt: null }; + if (verdict.kind === "settling") return { verdict, outcome: "settling", retryAt: verdict.recheckAt }; + if (verdict.stale) return { verdict, outcome: "stale_reported", retryAt: null }; + // A task blocked under an active recovery hold is re-continued by the + // hold's owner (the fresh-continuation fallback, same per-task cap). The + // sweep reports it and repairs only blocks that have no hold at all, so + // the two never continue one task twice. + if (issue.status === "blocked" && facts.activeRecoveryAction) return { verdict, outcome: "left_to_hold_owner", retryAt: null }; + // A stalled native run is ended by native recovery, not here: the sweep + // builds no second ender. It stays a counted violation in the report. + if (verdict.reason === "native_run_stalled") return { verdict, outcome: "left_to_native_recovery", retryAt: null }; + if (verdict.reason === "run_owner_gone") { + // Hand the run to heartbeat's lost-process settle path; its own failure + // policy schedules the continuation. Re-check after the grace. + await deps.settleLostRunProcess?.({ companyId: issue.companyId, runId: verdict.runId! }); + return { verdict, outcome: "settled_lost_run", retryAt: new Date(at.getTime() + 90_000) }; + } + // A small process-wide budget keeps a post-deploy backlog from starting + // hundreds of runs at once. Deferred tasks come back on their deadline. + const minuteAgo = at.getTime() - 60_000; + while (repairTimes.length && repairTimes[0] <= minuteAgo) repairTimes.shift(); + if (repairTimes.length >= ASSIGNED_WORK_REPAIRS_PER_MINUTE) { + return { verdict, outcome: "deferred_rate_limit", retryAt: new Date(repairTimes[0] + 60_000 + Math.floor(Math.random() * 5_000)) }; + } + const claimed = await claimContinuation(issue, at); + if (claimed.kind !== "claimed") return { verdict, outcome: claimed.kind === "capped" ? "deferred_cap" : claimed.kind === "changed" ? "changed" : "deferred_rate_limit", + retryAt: claimed.kind === "changed" ? null : claimed.retryAt }; + repairTimes.push(at.getTime()); + // A block that interrupted a review goes back to review; the review's own + // recovery wakes its participant. No agent work starts from here. + if (claimed.restoredTo === "in_review") { + await settleContinuationActivity(claimed, claimed.reason, "restored_to_review"); + return { verdict: claimed.verdict, outcome: "repaired", retryAt: null }; + } + const run = await deps.enqueueWakeup(issue.assigneeAgentId!, { + source: facts.latestRun ? "automation" : "assignment", triggerDetail: "system", + reason: facts.latestRun ? "issue_continuation_needed" : "issue_assigned", + payload: { issueId: issue.id, mutation: "assigned_work_invariant" }, + requestedByActorType: "system", requestedByActorId: null, + contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: facts.latestRun ? "issue_continuation_needed" : "issue_assigned", + source: ASSIGNED_WORK_CONTINUATION_SOURCE, assignedWorkViolation: claimed.reason, + ...(facts.latestRun ? { previousRunId: facts.latestRun.id } : {}) }, + }).catch(error => { + logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id }, "assigned-work continuation was not admitted"); + return null; + }); + // A refused continuation stays a violation; the claim spaces the next try. + // A block lifted for it is put back, so the task never sits unblocked + // with nothing pending. + if (!run && claimed.restoredTo) await revertRestoredBlock(issue, claimed); + await settleContinuationActivity(claimed, claimed.reason, run ? "queued" : "not_started"); + return { verdict: claimed.verdict, outcome: run ? "repaired" : "enqueue_refused", retryAt: claimed.nextCheckAt }; + } + + type Claimed = { kind: "claimed"; nextCheckAt: Date; activityId: string; publications: ActivityPublication[]; + verdict: Extract; reason: ViolationReason; + restoredTo: "todo" | "in_progress" | "in_review" | null; restoredStatusVersion: number | null; + blockedTransitionAt: Date | null; unblockDescriptor: IssueRow["unblockDescriptor"]; blockedOwnerNotifiedAt: Date | null }; + + /** + * The durable, serialized claim. Under the task's row lock it loads the + * task's facts again and re-runs the same predicate, so a question, approval, + * blocker, monitor, wake or run that appeared since the first read stops the + * repair. It then applies the per-task cap and spacing, lifts a block that has + * no hold, and records the activity entry. A second server sweeping at the + * same time waits on the lock and then sees the claim, so one violation + * produces one continuation. + */ + async function claimContinuation(issue: IssueRow, at: Date): + Promise { + const publications: ActivityPublication[] = []; + return db.transaction(async tx => { + const [current] = await tx.select(issueColumns).from(issues) + .where(and(eq(issues.companyId, issue.companyId), eq(issues.id, issue.id))).for("update"); + if (!current || current.status !== issue.status || current.statusVersion !== issue.statusVersion + || current.assigneeAgentId !== issue.assigneeAgentId || current.assigneeUserId || current.hiddenAt) return { kind: "changed" as const }; + const fresh = (await loadFacts(issue.companyId, [current], { database: tx as unknown as Db, proveOwners: false })).get(current.id)!; + const verdict = evaluateAssignedWork(fresh, at); + if (verdict.kind !== "violation" || verdict.stale || verdict.reason === "run_owner_gone" || verdict.reason === "native_run_stalled" + || (current.status === "blocked" && fresh.activeRecoveryAction)) return { kind: "changed" as const }; + const recent = await tx.select({ createdAt: activityLog.createdAt }).from(activityLog).where(and( + eq(activityLog.companyId, issue.companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issue.id), + eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION), + gte(activityLog.createdAt, new Date(at.getTime() - CONTINUATION_CAP_WINDOW_MS)))).orderBy(desc(activityLog.createdAt)).limit(64); + const allowance = continuationAllowance(recent.map(row => row.createdAt), at); + if (!allowance.allowed) return { kind: allowance.capped ? "capped" as const : "spaced" as const, retryAt: allowance.retryAt }; + let restoredTo: Claimed["restoredTo"] = null; + let restoredStatusVersion: number | null = null; + if (current.status === "blocked") { + // A recovery outcome with no hold left: nothing names a person's + // move. Put the task back where the block found it. + const [blocking] = await tx.select({ details: activityLog.details }).from(activityLog).where(and( + eq(activityLog.companyId, issue.companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issue.id), + sql`(${activityLog.details}->>'status' = 'blocked' or ${activityLog.details}->>'toStatus' = 'blocked')`, + )).orderBy(desc(activityLog.createdAt)).limit(1); + restoredTo = recordedPreviousStatus(blocking?.details); + await issueService(tx as unknown as Db).update(issue.id, { status: restoredTo }, tx); + const [restored] = await tx.select({ statusVersion: issues.statusVersion }).from(issues).where(eq(issues.id, issue.id)); + restoredStatusVersion = restored?.statusVersion ?? null; + } + const activity = await logActivity(tx as unknown as Db, { + companyId: issue.companyId, actorType: "system", actorId: "system", agentId: issue.assigneeAgentId, runId: null, + action: ASSIGNED_WORK_CONTINUED_ACTION, entityType: "issue", entityId: issue.id, + details: { identifier: issue.identifier, message: CONTINUATION_PENDING_MESSAGE, outcome: "pending", violation: verdict.reason, + ...(restoredTo ? { status: restoredTo, previousStatus: "blocked" } : {}), + attempt: recent.length + 1 }, + }, publications); + const next = continuationAllowance([at, ...recent.map(row => row.createdAt)], at); + return { kind: "claimed" as const, activityId: activity.id, publications, verdict, reason: verdict.reason, + restoredTo, restoredStatusVersion, blockedTransitionAt: current.blockedTransitionAt, + unblockDescriptor: current.unblockDescriptor, blockedOwnerNotifiedAt: current.blockedOwnerNotifiedAt, + nextCheckAt: next.allowed ? new Date(at.getTime() + 90_000) : next.retryAt }; + }); + } + + /** Put back a block lifted for a continuation that admission refused, when + * nothing changed the task since: same status version, nothing pending. */ + async function revertRestoredBlock(issue: IssueRow, claimed: Claimed) { + await db.transaction(async tx => { + const [current] = await tx.select(issueColumns).from(issues) + .where(and(eq(issues.companyId, issue.companyId), eq(issues.id, issue.id))).for("update"); + if (!current || current.status !== claimed.restoredTo || current.statusVersion !== claimed.restoredStatusVersion) return; + const [pendingWake] = await tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, issue.companyId), inArray(agentWakeupRequests.status, PENDING_WAKE_STATUSES), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`)).limit(1); + const [activeRun] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, issue.companyId), inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`)).limit(1); + if (pendingWake || activeRun) return; + await issueService(tx as unknown as Db).update(issue.id, { status: "blocked" }, tx); + // Keep the original block record: its time (the stale window counts + // from it), its declared unblock action, and that its owner was told. + await tx.update(issues).set({ blockedTransitionAt: claimed.blockedTransitionAt, unblockDescriptor: claimed.unblockDescriptor, + blockedOwnerNotifiedAt: claimed.blockedOwnerNotifiedAt }).where(eq(issues.id, issue.id)); + }); + } + + /** The claim's activity entry says what happened only once the wake was + * admitted or refused; it is published then, never before. A refused + * continuation stays a violation and is tried again under the same cap. */ + async function settleContinuationActivity(claimed: { activityId: string; publications: ActivityPublication[] }, + reason: ViolationReason, outcome: "queued" | "not_started" | "restored_to_review") { + const update = outcome === "queued" ? { outcome, message: continuationActivityMessage(reason) } + : outcome === "restored_to_review" ? { outcome, message: CONTINUATION_REVIEW_MESSAGE } + : { outcome, message: CONTINUATION_REFUSED_MESSAGE }; + await db.update(activityLog).set({ details: sql`coalesce(${activityLog.details}, '{}'::jsonb) || ${JSON.stringify(update)}::jsonb` }) + .where(eq(activityLog.id, claimed.activityId)); + for (const publication of claimed.publications) { + const details = json(publication.payload.details); + publishActivity({ ...publication, payload: { ...publication.payload, details: { ...details, ...update } } }); + } + } + + /** Read-only: classify the open assigned tasks of a company that the caller + * may read (`readCondition`, the caller's issue-privacy predicate). Private + * tasks the caller cannot read are left out of every count and sample. The + * sweep itself evaluates all tasks. */ + async function report(companyId: string, options: { readCondition?: SQL } = {}) { + const rows = await loadIssues(companyId, undefined, ASSIGNED_WORK_REPORT_LIMIT + 1, options.readCondition); + const truncated = rows.length > ASSIGNED_WORK_REPORT_LIMIT; + if (truncated) rows.length = ASSIGNED_WORK_REPORT_LIMIT; + // When the read was capped, say how many open tasks exist, and that the + // violation count is a lower bound: it covers only the newest tasks read. + const [total] = truncated ? await db.select({ count: sql`count(*)::int` }).from(issues).where(and( + eq(issues.companyId, companyId), options.readCondition, inArray(issues.status, [...OPEN_STATUSES]), + sql`${issues.assigneeAgentId} is not null`, isNull(issues.assigneeUserId), isNull(issues.hiddenAt))) : [{ count: rows.length }]; + const at = now(); + const summary = { checked: rows.length, openTasks: Number(total?.count ?? rows.length), truncated, + violationCountIsLowerBound: truncated, ok: {} as Record, + violations: {} as Record, violationCount: 0, heldByRecoveryOwner: 0, staleBlocked: 0, settling: 0, + sample: [] as Array<{ identifier: string | null; status: string; reason: string }> }; + for (let index = 0; index < rows.length; index += 200) { + const batch = rows.slice(index, index + 200); + const facts = await loadFacts(companyId, batch); + for (const issue of batch) { + const verdict = evaluateAssignedWork(facts.get(issue.id)!, at); + if (verdict.kind === "ok") { + const key = verdict.clause === "live_run" ? "live_run" : verdict.clause === "pending_intent" ? `pending:${verdict.intent}` : `waiting:${verdict.reason}`; + summary.ok[key] = (summary.ok[key] ?? 0) + 1; + } else if (verdict.kind === "settling") summary.settling += 1; + else if (verdict.kind === "violation") { + if (verdict.stale) { summary.staleBlocked += 1; continue; } + summary.violationCount += 1; + if (issue.status === "blocked" && facts.get(issue.id)!.activeRecoveryAction) summary.heldByRecoveryOwner += 1; + summary.violations[verdict.reason] = (summary.violations[verdict.reason] ?? 0) + 1; + if (summary.sample.length < 20) summary.sample.push({ identifier: issue.identifier, status: issue.status, reason: verdict.reason }); + } + } + } + // Agents whose slots are held by native runs no live owner renews: their + // queued work never dispatches, the same failure seen from the agent. + // Every count passes through the caller's issue-read filter: runs are + // joined to their task, so a private task's runs are never counted. + const stallCutoff = new Date(at.getTime() - NATIVE_STALL_MS); + const stalledRows = await db.select({ agentId: agents.id, agentName: agents.name, + stalledRuns: sql`count(distinct ${heartbeatRuns.id})::int` }).from(heartbeatRuns) + .innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id)) + .innerJoin(agents, eq(agents.id, heartbeatRuns.agentId)) + .innerJoin(issues, and(eq(issues.id, nativeRunFinalizations.issueId), eq(issues.companyId, heartbeatRuns.companyId))) + .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.runtimeMode, "native"), + isNull(nativeRunFinalizations.resultId), sql`${nativeRunFinalizations.recoveryState} is not null`, + or(isNull(nativeRunFinalizations.leaseExpiresAt), sql`${nativeRunFinalizations.leaseExpiresAt} < ${stallCutoff.toISOString()}::timestamptz`), + sql`${nativeRunFinalizations.updatedAt} < ${stallCutoff.toISOString()}::timestamptz`, options.readCondition)) + .groupBy(agents.id, agents.name).orderBy(desc(sql`count(distinct ${heartbeatRuns.id})`)).limit(20); + const queuedRows = stalledRows.length ? await db.select({ agentId: heartbeatRuns.agentId, queuedRuns: sql`count(*)::int` }) + .from(heartbeatRuns).innerJoin(issues, and(eq(issues.companyId, heartbeatRuns.companyId), + sql`${issues.id}::text = ${heartbeatRuns.contextSnapshot}->>'issueId'`)) + .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "queued"), + inArray(heartbeatRuns.agentId, stalledRows.map(row => row.agentId)), options.readCondition)) + .groupBy(heartbeatRuns.agentId) : []; + const queuedByAgent = new Map(queuedRows.map(row => [row.agentId, Number(row.queuedRuns)])); + const slotRows = stalledRows.map(row => ({ agentName: row.agentName, stalledRuns: Number(row.stalledRuns), + queuedRuns: queuedByAgent.get(row.agentId) ?? 0 })); + return { ...summary, agentsWithStalledSlots: slotRows }; + } + + return { checkIssue, listSuspects, report, loadFacts, flushPassLog: passLog.flush }; +} + +/** Who moved a task to `blocked`, from the activity entry written with it. */ +export function attributeBlock(entry: { actorType: string; actorId: string; agentId: string | null; details: unknown } | undefined, + assigneeAgentId: string | null): "user" | "assignee" | "system" | "unknown" { + if (!entry) return "unknown"; + const details = json(entry.details); + if (entry.actorType === "user") return "user"; + if (entry.actorType === "agent") return entry.agentId && entry.agentId === assigneeAgentId ? "assignee" : "system"; + // The agent's own turn decided to block (committed by the native status + // decision), except when the decision only bound a dependency. + if (entry.actorId === "native-status-committer" && details.source === "native_status_decision" + && details.reasonCode !== "durable_dependency_blocker_bound") return "assignee"; + return "system"; +} + +/** A pending interaction no explicit user continuation (Retry or message) + * started after it. Uses the (company, issueId, created) run index. */ +function interactionNotSupersededSql(companyId: SQLWrapper, issueIdText: SQLWrapper, createdAt: SQLWrapper) { + return sql`not exists (select 1 from heartbeat_runs retry where retry.company_id = ${companyId} + and (retry.context_snapshot->>'issueId') = ${issueIdText} + and jsonb_typeof(retry.context_snapshot->'explicitUserContinuation') = 'object' + and retry.created_at > ${createdAt})`; +} + +/** The status a block replaced, from the activity entry that blocked the task: + * the normal update route records it as `_previous.status`, recovery as + * `previousStatus`, the native status committer as `fromStatus`. */ +export function recordedPreviousStatus(details: unknown): "todo" | "in_progress" | "in_review" { + const record = json(details); + const candidates = [json(record._previous).status, record.previousStatus, record.fromStatus]; + const recorded = candidates.find(value => value === "todo" || value === "in_progress" || value === "in_review"); + return (recorded as "todo" | "in_progress" | "in_review" | undefined) ?? "todo"; +} diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 424f0efc47f..7ea066bc377 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -33,6 +33,7 @@ import { externalConversationStateSql } from "../slack-conversation-state.js"; import { isIssueReviewPathRecoveryRun } from "./review-path-recovery.js"; import { escalateExhaustedIssueReviewPathRecovery } from "./review-path-recovery-escalation.js"; import { executionRetryAccounting, INTERRUPTED_PREPARATION_RETRY_REASON } from "../execution-recovery-attempt.js"; +import { assignedWorkSweep } from "./assigned-work-sweep.js"; import { isExplicitContinuationRetryClaim } from "../explicit-continuation-retry-claim.js"; import { decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint, @@ -6813,7 +6814,17 @@ export function recoveryService( return result; } + // The assigned-work invariant: the backstop that leaves no open assigned + // task with nothing running, nothing pending and no one else's move. The + // stranded sweep above still decides how a known failure resumes. + const assignedWork = assignedWorkSweep(db, { + enqueueWakeup: (agentId, opts) => deps.enqueueWakeup(agentId, opts), + liveRunExecutions: deps.liveRunExecutions, + settleLostRunProcess: deps.settleLostRunProcess, + }); + return { + assignedWork, buildRunOutputSilence, escalateStrandedRecoveryIssueInPlace, escalateStrandedAssignedIssue, diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index dec58947ca1..2637f65f30f 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -22,6 +22,12 @@ export function registerIssueRecoveryWork( locks(scope: Scope): Promise; watchdogs(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>; schedulingSuppressed(): Promise; + /** The assigned-work invariant (assigned-work-sweep.ts). */ + assignedWork?: { + listSuspects(scope: { companyId?: string; issueId?: string }, cursor: string | null, limit: number): + Promise>; + checkIssue(scope: Scope): Promise; + }; }, canRun: () => boolean, ) { @@ -60,6 +66,39 @@ export function registerIssueRecoveryWork( }, }, canRun); } + /** Event-driven: run terminal states, hold settlements and boot mark the + * company dirty; checks schedule their own grace and back-off deadlines. + * Newest-updated tasks are checked first, a small batch per pass. */ + function registerAssignedWork(sweep: NonNullable) { + type Suspect = Entry & { updatedAt: number }; + const cursors = new Map(); + let enabled = false; + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.assignedWorkInvariant, + reconstructionIntervalMs: 30_000, + batchSize: 25, + beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope, + matches: (entry, scope) => !scope || (entry.companyId === scope.companyId && (!scope.issueId || entry.id === scope.issueId)), + priority: entry => entry.updatedAt, + mergeEntry: (pending, loaded) => ({ ...loaded, dueAt: Math.max(pending.dueAt, loaded.dueAt) }), + load: async scope => { + const key = scope ? JSON.stringify([scope.companyId, scope.issueId]) : "*"; + if (!enabled) { cursors.delete(key); return []; } + const rows = await sweep.listSuspects(scope ?? {}, cursors.get(key) ?? null, PAGE_SIZE); + const complete = rows.length < PAGE_SIZE; + if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now() })), complete }; + }, + check: async entry => { + if (!enabled) return null; + let next: number | null = null; + await sweep.checkIssue({ companyId: entry.companyId, issueId: entry.id, + onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); + return next === null ? null : { ...entry, dueAt: next }; + }, + }, canRun); + } function registerCompany(queue: DeliveryQueue, kind: "blockers" | "actions", check: (scope: { companyId: string }) => Promise) { const cursors = new Map(); let enabled = false; @@ -117,11 +156,13 @@ export function registerIssueRecoveryWork( if (result.enqueueFailed > 0) throw new Error("Resolved dependency wake remains undelivered"); }); const locks = register(DELIVERY_QUEUES.staleIssueLocks, null, owner.locks); + const assignedWork = owner.assignedWork ? registerAssignedWork(owner.assignedWork) : null; const watchdogs = registerTaskWatchdogWork(db, coordinator, { reconcile: owner.watchdogs, enabled: async () => !(await owner.schedulingSuppressed()), }, canRun); - const workers = [assignments, orphanBlockers, recoveryActions, dependencies, locks, watchdogs]; - return { watchdogs, assignments, ready: Promise.all(workers.map(worker => worker.ready)), + const workers = [assignments, orphanBlockers, recoveryActions, dependencies, locks, watchdogs, + ...(assignedWork ? [assignedWork] : [])]; + return { watchdogs, assignments, assignedWork, ready: Promise.all(workers.map(worker => worker.ready)), wake() { for (const worker of workers) worker.invalidate(); }, async stop() { await Promise.all(workers.map(worker => worker.stop())); } }; }