diff --git a/doc/heartbeat-internals.md b/doc/heartbeat-internals.md index 59be8eafd62..174f167cd51 100644 --- a/doc/heartbeat-internals.md +++ b/doc/heartbeat-internals.md @@ -56,6 +56,50 @@ orphan recovery. pnpm exec vitest run server/src/services/heartbeat/agent-status.test.ts server/src/__tests__/heartbeat-agent-status-recovery.test.ts ``` +## Queued run dispatch order + +When an agent is at its concurrency limit, each freed slot starts the next +queued run chosen by `server/src/services/heartbeat/queued-run-dispatch-order.ts`. +The "place in line" a waiting task shows uses the same facts and order +(`queued-run-line.ts`). + +Runs fall into four tiers, started in this order: + +1. **Person.** A run a person asked for (an assignment, comment, chat message + or Retry by a board user, itself or through its source chain), and any other + non-recovery run on a task tree a person created. A task belongs to such a + tree when it, or any task above it, has `createdByUserId`. This covers an + agent's subtasks of a person's task and the parent's resume once they + complete (`issue_children_completed`), so a half-finished task is not left + behind newer tasks. +2. **Work.** Other agent work, such as a tree an agent or routine started. +3. **Recovery** and 4. **old recovery backlog.** System recovery wakes, even on + a person's tree. + +Within a tier, runs are ordered by readiness (in progress first), then task +priority, then age, oldest first. A newer run on a person's tree never passes +another person's equally ranked task that has waited longer. + +Fairness bounds: + +- **Oldest-run share.** Every `OLDEST_DISPATCH_SHARE` (4th) start goes to the + longest-waiting ready run that has waited past `QUEUED_RUN_AGING_MS` + (60 minutes), preferring runs people wait on. So even while a deep tree keeps + adding in-progress or higher-priority work, another person's new task starts + within 60 minutes plus 4 x (aged person-tier runs queued before it + 1) + starts. +- **Recovery share.** Recovery takes every `RECOVERY_DISPATCH_SHARE` (4th) + start from lower-tier work. It is held only while a run a person asked for is + ready, not for the rest of a person's tree, so a deep tree cannot hold + recovery indefinitely. +- **Scope.** Queues are per agent, and agents belong to one company, so tree + priority never moves one agent's or company's runs ahead of another's. + +Tree membership is read in one recursive query per dispatch pass, walking each +queued task's parent chain by the `(company_id, id)` unique index and stopping +at the first person-created task (at most 32 levels). It runs only when the +agent has more than one queued run. + ## Run history authorization reads Run history reuses the authorization service's batch read API for distinct task diff --git a/server/src/__tests__/heartbeat-person-tree-dispatch.test.ts b/server/src/__tests__/heartbeat-person-tree-dispatch.test.ts new file mode 100644 index 00000000000..f3fa82443c8 --- /dev/null +++ b/server/src/__tests__/heartbeat-person-tree-dispatch.test.ts @@ -0,0 +1,158 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agentWakeupRequests, agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issues } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { heartbeatService } from "../services/heartbeat.ts"; +import { runningProcesses } from "../adapters/index.ts"; +import { automaticRunBrake } from "../services/recovery/automatic-run-brake.js"; +import { logger } from "../middleware/logger.js"; + +const gate = vi.hoisted(() => ({ open: false, releases: [] as Array<() => void> })); + +const mockAdapterExecute = vi.hoisted(() => vi.fn(async () => { + if (!gate.open) await new Promise(resolve => gate.releases.push(resolve)); + return { exitCode: 0, signal: null, timedOut: false, errorMessage: null, summary: "Person tree dispatch test run.", + provider: "test", model: "test-model" }; +})); + +vi.mock("../adapters/index.ts", async () => { + const actual = await vi.importActual("../adapters/index.ts"); + return { ...actual, getServerAdapter: vi.fn(() => ({ supportsLocalAgentJwt: false, execute: mockAdapterExecute })) }; +}); + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; +const MINUTE = 60_000; + +suite("dispatch of work on a task tree a person created, through the heartbeat queue", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + let heartbeat!: ReturnType; + const userId = randomUUID(); + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-person-tree-dispatch-"); + db = createDb(temporary.connectionString); + heartbeat = heartbeatService(db); + await db.insert(authUsers).values({ id: userId, name: "Board", email: `${userId}@example.test`, emailVerified: true, + createdAt: new Date(), updatedAt: new Date() }); + }, 60_000); + + afterEach(async () => { + gate.open = true; + for (const release of gate.releases.splice(0)) release(); + await heartbeat.drainActiveRunExecutions(); + gate.open = false; + mockAdapterExecute.mockClear(); + runningProcesses.clear(); + automaticRunBrake.reset(); + await db.execute(sql`truncate table ${companies} cascade`); + }, 180_000); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }, 120_000); + + async function seed() { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Person tree", status: "active", + issuePrefix: `T${companyId.replaceAll("-", "").slice(0, 6).toUpperCase()}`, requireBoardApprovalForNewAgents: false, + defaultResponsibleUserId: userId }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", + membershipRole: "owner" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Codie", role: "engineer", status: "idle", + adapterType: "codex_local", adapterConfig: {}, permissions: {}, + runtimeConfig: { heartbeat: { enabled: true, intervalSec: 60, wakeOnDemand: true, maxConcurrentRuns: 1 } } }); + const [busy] = await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", + triggerDetail: "system", status: "running", createdAt: new Date(Date.now() - 90 * MINUTE), + startedAt: new Date(Date.now() - 90 * MINUTE) }).returning(); + const task = async (input: { title: string; parentId?: string; byPerson: boolean; status?: string }) => { + const [row] = await db.insert(issues).values({ companyId, title: input.title, status: input.status ?? "todo", + assigneeAgentId: agentId, parentId: input.parentId ?? null, createdByUserId: input.byPerson ? userId : null, + createdByAgentId: input.byPerson ? null : agentId }).returning({ id: issues.id }); + return row!.id; + }; + const queue = async (issueId: string, wakeReason: string, ageMinutes: number, byPerson = false) => { + const createdAt = new Date(Date.now() - ageMinutes * MINUTE); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId, scopeKind: "issue" as const, + invocationSource: byPerson ? "assignment" : "automation", triggerDetail: byPerson ? "manual" : "system", + status: "queued", createdAt, contextSnapshot: { wakeReason, issueId, taskId: issueId } }).returning(); + if (byPerson) { + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "assignment", reason: wakeReason, + status: "queued", requestedByActorType: "user", requestedByActorId: userId, runId: run!.id, requestedAt: createdAt, + createdAt }).returning({ id: agentWakeupRequests.id }); + await db.update(heartbeatRuns).set({ wakeupRequestId: wake!.id }).where(eq(heartbeatRuns.id, run!.id)); + } + return run!.id; + }; + const freeSlot = async () => { + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, busy!.id)); + await heartbeat.resumeQueuedRunsAfterDrain({ companyId }); + }; + return { companyId, agentId, task, queue, freeSlot }; + } + + const statusOf = async (ids: string[]) => new Map((await db.select({ id: heartbeatRuns.id, status: heartbeatRuns.status, + startedAt: heartbeatRuns.startedAt }).from(heartbeatRuns).where(inArray(heartbeatRuns.id, ids))).map(row => [row.id, row])); + + async function expectStartsFirst(f: Awaited>, first: string, waiting: string) { + await heartbeat.resumeQueuedRunsAfterDrain({ companyId: f.companyId }); + expect([...(await statusOf([first, waiting])).values()].every(run => run.status === "queued")).toBe(true); + await f.freeSlot(); + await vi.waitFor(async () => expect((await statusOf([first])).get(first)?.status).toBe("running"), { timeout: 20_000 }); + const runs = await statusOf([first, waiting]); + expect(runs.get(first)?.startedAt).toBeInstanceOf(Date); + expect(runs.get(waiting)).toMatchObject({ status: "queued", startedAt: null }); + } + + it("starts an agent's subtask of a person's task before a newer assignment by a person when a slot frees", async () => { + const f = await seed(); + const root = await f.task({ title: "Person's task", byPerson: true, status: "blocked" }); + const subtask = await f.task({ title: "Agent's subtask", parentId: root, byPerson: false, status: "blocked" }); + const grandchild = await f.task({ title: "Agent's sub-subtask", parentId: subtask, byPerson: false }); + const subtaskRun = await f.queue(grandchild, "issue_assigned", 40); + const newAssignment = await f.queue(await f.task({ title: "New task", byPerson: true }), "issue_assigned", 5, true); + await expectStartsFirst(f, subtaskRun, newAssignment); + }, 120_000); + + it("starts a person's parent task's resume after its children complete before a newer assignment by a person", async () => { + const f = await seed(); + const parent = await f.task({ title: "Person's task", byPerson: true }); + await f.task({ title: "Done subtask", parentId: parent, byPerson: false, status: "done" }); + const resume = await f.queue(parent, "issue_children_completed", 30); + const newAssignment = await f.queue(await f.task({ title: "New task", byPerson: true }), "issue_assigned", 5, true); + await expectStartsFirst(f, resume, newAssignment); + }, 120_000); + + it("keeps an agent-started tree's subtask behind a newer assignment by a person", async () => { + const f = await seed(); + const root = await f.task({ title: "Agent's task", byPerson: false, status: "blocked" }); + const subtaskRun = await f.queue(await f.task({ title: "Agent's subtask", parentId: root, byPerson: false }), "issue_assigned", 40); + const newAssignment = await f.queue(await f.task({ title: "New task", byPerson: true }), "issue_assigned", 5, true); + await expectStartsFirst(f, newAssignment, subtaskRun); + }, 120_000); + + it("backs off a refused run on a person's tree like agent work, and offers it again once the backoff passes", async () => { + const f = await seed(); + await db.update(companies).set({ defaultResponsibleUserId: null }).where(eq(companies.id, f.companyId)); + await db.delete(companyMemberships).where(eq(companyMemberships.companyId, f.companyId)); + const root = await f.task({ title: "Person's task", byPerson: true, status: "blocked" }); + const refused = await f.queue(await f.task({ title: "Agent's subtask", parentId: root, byPerson: false }), "issue_assigned", 180); + await f.queue(await f.task({ title: "Another subtask", parentId: root, byPerson: false }), "issue_assigned", 170); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }) + .where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "running"))); + const warn = vi.spyOn(logger, "warn"); + const offers = () => warn.mock.calls.filter(([fields, message]) => typeof message === "string" + && message.startsWith("long-queued heartbeat run was offered a free slot") + && (fields as { runId?: string }).runId === refused).length; + await heartbeat.resumeQueuedRunsForAgent(f.companyId, f.agentId); + expect(offers()).toBe(1); + await new Promise(resolve => setTimeout(resolve, 3_000)); + await heartbeat.resumeQueuedRunsForAgent(f.companyId, f.agentId); + expect(offers()).toBe(1); + await vi.waitFor(async () => { + await heartbeat.resumeQueuedRunsForAgent(f.companyId, f.agentId); + expect(offers()).toBe(2); + }, { timeout: 20_000, interval: 1_000 }); + expect((await statusOf([refused])).get(refused)?.status).toBe("queued"); + }, 120_000); +}); diff --git a/server/src/services/execution-projection-agent-queue.test.ts b/server/src/services/execution-projection-agent-queue.test.ts index ea86124347b..81dc9ce290f 100644 --- a/server/src/services/execution-projection-agent-queue.test.ts +++ b/server/src/services/execution-projection-agent-queue.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; -import { agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db"; +import { agentWakeupRequests, agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { executionProjectionsForRuns } from "./execution-projection.js"; import { agentQueuedRunLine, clearAgentQueueLineCache, worktreeExecutionCutoff } from "./heartbeat/queued-run-line.js"; @@ -146,6 +146,38 @@ const minutesAgo = (minutes: number) => new Date(now.getTime() - minutes * 60_00 expect(elapsedMs).toBeLessThan(5_000); }); + it("starts work on a task tree a person created ahead of a newer person assignment, and leaves an agent's tree behind it", async () => { + const f = await fixture({ maxConcurrentRuns: 1, running: 1 }); + const task = async (input: { createdByUserId?: string; parentId?: string }) => { + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId: f.companyId, title: "Task", assigneeAgentId: f.agentId, status: "todo", + priority: "medium", parentId: input.parentId ?? null, createdByUserId: input.createdByUserId ?? null, + createdByAgentId: input.createdByUserId ? null : f.agentId }); + return issueId; + }; + const queueOn = async (issueId: string, wakeReason: string, createdMinutesAgo: number, byPerson = false) => { + const runId = randomUUID(); + const wakeupRequestId = byPerson ? randomUUID() : null; + if (wakeupRequestId) await db.insert(agentWakeupRequests).values({ id: wakeupRequestId, companyId: f.companyId, + agentId: f.agentId, source: "assignment", reason: wakeReason, requestedByActorType: "user", requestedByActorId: "board-user" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, status: "queued", issueId, + wakeupRequestId, createdAt: minutesAgo(createdMinutesAgo), contextSnapshot: { issueId, wakeReason } }); + return runId; + }; + const personRoot = await task({ createdByUserId: "board-user" }); + const child = await task({ parentId: personRoot }); + const grandchild = await task({ parentId: child }); + const agentRoot = await task({}); + const agentChild = await task({ parentId: agentRoot }); + const agentTreeWork = await queueOn(agentChild, "issue_assigned", 50); + const subtask = await queueOn(grandchild, "issue_assigned", 40); + const parentResume = await queueOn(personRoot, "issue_children_completed", 30); + const newAssignment = await queueOn(await task({ createdByUserId: "board-user" }), "issue_assigned", 5, true); + + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + expect(await agentQueuedRunLine(db, issueService(db), agent!, now)).toEqual([subtask, parentResume, newAssignment, agentTreeWork]); + }); + it("leaves running runs and other companies' runs out of the line", async () => { const f = await fixture({ maxConcurrentRuns: 1, running: 1 }); const mine = await f.queue({ priority: "medium", createdMinutesAgo: 5 }); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 74b219eb610..f104d29a928 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -12,7 +12,7 @@ import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, reche import { deferredWorkIntentWait } from "../work-intent-outbox.js"; import { notifyIssueGraphRecovery } from "../issue-recovery-notifications.js"; import { createHeartbeatTimerPolicy } from "./timer-policy.js"; -import { QUEUED_RUN_AGING_MS, RECOVERY_WAKE_REASONS, createQueuedRunDispatcher, hasQueuedRunAged, queuedRunDispatchClass } from "./queued-run-dispatch-order.js"; +import { QUEUED_RUN_AGING_MS, RECOVERY_WAKE_REASONS, createQueuedRunDispatcher, hasQueuedRunAged, isPersonRequestedRun, queuedRunDispatchClass } from "./queued-run-dispatch-order.js"; import { reserveAutomaticRunStart } from "../recovery/automatic-run-brake.js"; import { queuedRunDispatchFacts } from "./queued-run-line.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; @@ -2527,7 +2527,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende claimed: (run: QueuedRunCandidate) => { const entry = byRun.get(run); if (entry) dispatcher.claimed(entry); }, personRun: (run: QueuedRunCandidate) => { const entry = byRun.get(run); - return Boolean(entry && queuedRunDispatchClass(entry, now) === "person"); + return Boolean(entry && queuedRunDispatchClass(entry, now) === "person" && isPersonRequestedRun(entry)); }, personOrigin: (run: QueuedRunCandidate) => personOriginChecked ? byRun.get(run)?.requestedByUser ?? false : undefined, }; diff --git a/server/src/services/heartbeat/queued-run-dispatch-order.test.ts b/server/src/services/heartbeat/queued-run-dispatch-order.test.ts index 3ae58093c2d..c29305247ed 100644 --- a/server/src/services/heartbeat/queued-run-dispatch-order.test.ts +++ b/server/src/services/heartbeat/queued-run-dispatch-order.test.ts @@ -208,6 +208,80 @@ describe("queued run dispatch order", () => { }); }); +describe("work on a task tree a person created", () => { + const newAssignment = (createdMinutesAgo: number) => run({ wakeReason: "issue_assigned", requestedByUser: true, + readinessRank: 1, createdAt: minutesAgo(createdMinutesAgo) }); + const first = (queue: QueuedRunDispatchFacts[]) => { + const input = { now, startsSinceRecovery: 0, startsSinceOldest: 0 }; + const [next] = orderQueuedRunsForDispatch(queue, input); + expect(queuedRunDispatchLine(queue, input)[0]!.id).toBe(next!.id); + return next!; + }; + + it("starts the parent's resume after its children complete ahead of a newer assignment by a person", () => { + const resume = run({ wakeReason: "issue_children_completed", personTask: true, readinessRank: 1, createdAt: minutesAgo(20) }); + const assignment = newAssignment(10); + expect(queuedRunDispatchClass(resume, now)).toBe("person"); + expect(first([assignment, resume]).id).toBe(resume.id); + }); + + it("starts a subtask of a person's task ahead of a newer assignment by a person", () => { + const subtask = run({ wakeReason: "issue_assigned", personTask: true, readinessRank: 1, createdAt: minutesAgo(20) }); + const assignment = newAssignment(10); + expect(first([assignment, subtask]).id).toBe(subtask.id); + }); + + it("keeps work on a tree an agent started in the work tier, behind a newer assignment by a person", () => { + const agentTree = run({ wakeReason: "issue_children_completed", readinessRank: 1, createdAt: minutesAgo(20) }); + const assignment = newAssignment(10); + expect(queuedRunDispatchClass(agentTree, now)).toBe("work"); + expect(first([agentTree, assignment]).id).toBe(assignment.id); + }); + + it("keeps system recovery on a person's tree in the recovery tier", () => { + const recovery = run({ wakeReason: "issue_continuation_needed", personTask: true, sourceEndedAt: minutesAgo(5), + createdAt: minutesAgo(4) }); + expect(queuedRunDispatchClass(recovery, now)).toBe("recovery"); + }); + + it("starts another person's new task within the oldest-run share while a deep tree keeps adding in-progress work", () => { + const start = new Date(now.getTime() - 2 * 60 * 60_000); + const at = (minutes: number) => new Date(start.getTime() + minutes * 60_000); + let id = 0; + const treeRun = (minutes: number) => ({ id: `tree-${++id}`, wakeReason: "issue_commented", requestedByUser: false, + personTask: true, sourceEndedAt: null, readinessRank: 0, priorityRank: 1, createdAt: at(minutes) }); + const otherPerson = { ...newAssignment(0), createdAt: at(0) }; + let pending: QueuedRunDispatchFacts[] = [otherPerson, ...Array.from({ length: 10 }, () => treeRun(0))]; + const started: QueuedRunDispatchFacts[] = []; + // One start a minute, and the tree adds two higher-ranked runs for every one that starts. + for (let minute = 0; minute < 120 && !started.includes(otherPerson); minute += 1) { + pending.push(treeRun(minute), treeRun(minute)); + const clock = at(minute); + const history = [...started].reverse().map(entry => queuedRunStartOf(entry, clock)); + const [next] = orderQueuedRunsForDispatch(pending, { now: clock, ...dispatchCountersFromStarts(pending, history, clock) }); + started.push(next!); + pending = pending.filter(entry => entry.id !== next!.id); + } + const position = started.indexOf(otherPerson); + expect(position).toBeGreaterThanOrEqual(0); + expect(position).toBeLessThan(QUEUED_RUN_AGING_MS / 60_000 + OLDEST_DISPATCH_SHARE); + }); + + it("never lets newer tree work pass another person's waiting task of the same rank", () => { + const otherPerson = newAssignment(5); + const tree = Array.from({ length: 20 }, (_, index) => run({ wakeReason: "issue_assigned", personTask: true, readinessRank: 1, + createdAt: minutesAgo(4 - index * 0.1) })); + expect(first([...tree, otherPerson]).id).toBe(otherPerson.id); + }); + + it("does not hold recovery's share for tree work a person did not ask for", () => { + const recovery = run({ wakeReason: "issue_recovery_action_restored", createdAt: minutesAgo(3), sourceEndedAt: minutesAgo(4) }); + const tree = Array.from({ length: 5 }, () => run({ wakeReason: "issue_assigned", personTask: true, createdAt: minutesAgo(2) })); + const ordered = orderQueuedRunsForDispatch([...tree, recovery], { now, startsSinceRecovery: RECOVERY_DISPATCH_SHARE - 1 }); + expect(ordered[0]!.id).toBe(recovery.id); + }); +}); + describe("deep per-agent queue: a recovery burst behind 20 busy slots (paperclip.staging, 2026-11-13)", () => { // Codie, maxConcurrentRuns 20, every slot busy: each freed slot is one dispatch. const at = new Date("2026-11-13T14:30:00.000Z"); diff --git a/server/src/services/heartbeat/queued-run-dispatch-order.ts b/server/src/services/heartbeat/queued-run-dispatch-order.ts index 0aef1f561c5..8f8012faa7a 100644 --- a/server/src/services/heartbeat/queued-run-dispatch-order.ts +++ b/server/src/services/heartbeat/queued-run-dispatch-order.ts @@ -27,7 +27,10 @@ export type QueuedRunDispatchFacts = { readinessRank: number; priorityRank: number; shareDeferred?: boolean; - /** The task was created by a person, who waits on its outcome whichever system wake moves it. */ + /** + * The task, or a task above it, was created by a person, who waits on the + * whole tree's outcome whichever wake moves it. + */ personTask?: boolean; }; @@ -41,12 +44,16 @@ export function isSystemRecoveryRun(run: Pick) { + return run.requestedByUser || Boolean(run.wakeReason && PERSON_WAKE_REASONS.has(run.wakeReason)); +} + export function queuedRunDispatchClass(run: QueuedRunDispatchFacts, now: Date): QueuedRunDispatchClass { if (isSystemRecoveryRun(run)) { const endedAt = run.sourceEndedAt ?? run.createdAt; return now.getTime() - endedAt.getTime() > RECOVERY_BACKLOG_AGE_MS ? "recovery_backlog" : "recovery"; } - if (run.requestedByUser || (run.wakeReason && PERSON_WAKE_REASONS.has(run.wakeReason))) return "person"; + if (isPersonRequestedRun(run) || run.personTask) return "person"; return "work"; } @@ -59,7 +66,8 @@ const NOT_READY_TIER = 4; const isReady = (run: QueuedRunDispatchFacts) => run.readinessRank < NOT_READY_RANK; const isPersonRun = (run: QueuedRunDispatchFacts, now: Date) => queuedRunDispatchClass(run, now) === "person"; -/** A run a person asked for, or any run on a task a person created. */ +const holdsRecoveryShare = (run: QueuedRunDispatchFacts) => isReady(run) && !isSystemRecoveryRun(run) && isPersonRequestedRun(run); +/** A run a person asked for, or any run on a task tree a person created. */ const isPersonWaitingRun = (run: QueuedRunDispatchFacts, now: Date) => Boolean(run.personTask) || isPersonRun(run, now); const isOldestShareEligible = (run: QueuedRunDispatchFacts) => isReady(run) && !run.shareDeferred; @@ -107,7 +115,7 @@ export type QueuedRunStart = { wakeReason: string | null; createdAt: Date; start /** A dispatched run as the start history records it. */ export function queuedRunStartOf(run: QueuedRunDispatchFacts, now: Date): QueuedRunStart { - return { wakeReason: run.wakeReason, createdAt: run.createdAt, personOrigin: isPersonRun(run, now), + return { wakeReason: run.wakeReason, createdAt: run.createdAt, personOrigin: !isSystemRecoveryRun(run) && isPersonRequestedRun(run), personWaiting: isPersonWaitingRun(run, now) }; } @@ -185,8 +193,8 @@ export function createQueuedRunDispatcher( if (!remaining.length) return null; const oldest = sinceOldest >= OLDEST_DISPATCH_SHARE - 1 ? oldestShareCandidate(remaining, input.now) : null; if (oldest) return remaining.splice(remaining.indexOf(oldest), 1)[0]!; - // Recovery's share comes out of agent work, never out of a person's turn. - const personWaiting = remaining.some(run => isReady(run) && isPersonRun(run, input.now)); + // Recovery's share comes out of agent work, never out of the turn of a run a person asked for. + const personWaiting = remaining.some(holdsRecoveryShare); // The share goes to the longest-waiting recovery run once one has waited // past the aging bound, so fresh recovery cannot keep an old one waiting. const agedRecovery = oldestReadyQueuedRun(remaining, run => isSystemRecoveryRun(run) && hasQueuedRunAged(run, input.now)); @@ -254,7 +262,7 @@ export function queuedRunDispatchLine( const oldestRecovery = agedFirst(cursor(byAge(indexes(run => isOldestShareEligible(run) && isSystemRecoveryRun(run))))); const firstRecovery = cursor(indexes(run => isReady(run) && isSystemRecoveryRun(run))); const head = cursor(order.map((_, index) => index)); - let readyPeople = order.filter(run => isReady(run) && isPersonRun(run, now)).length; + let readyPeople = order.filter(holdsRecoveryShare).length; let sinceRecovery = input.startsSinceRecovery; let sinceOldest = input.startsSinceOldest ?? 0; const result: T[] = []; @@ -268,7 +276,7 @@ export function queuedRunDispatchLine( if (pick < 0) pick = head(); removed[pick] = 1; const run = order[pick]!; - if (isReady(run) && isPersonRun(run, now)) readyPeople -= 1; + if (holdsRecoveryShare(run)) readyPeople -= 1; result.push(run); sinceRecovery = isSystemRecoveryRun(run) ? 0 : sinceRecovery + 1; sinceOldest = candidate < 0 || servesOldestShare({ createdAt: run.createdAt, personWaiting: isPersonWaitingRun(run, now) }, diff --git a/server/src/services/heartbeat/queued-run-line.ts b/server/src/services/heartbeat/queued-run-line.ts index 573b7e3e348..955eb454f1a 100644 --- a/server/src/services/heartbeat/queued-run-line.ts +++ b/server/src/services/heartbeat/queued-run-line.ts @@ -90,6 +90,28 @@ export async function listPersonOriginSourceRunIds(db: Db, companyId: string, so return [...result].map(row => String(row.root_id)); } +const PERSON_TREE_ANCESTOR_DEPTH = 32; + +export async function listPersonTreeIssueIds(db: Db, companyId: string, issueIds: string[]): Promise> { + const ids = [...new Set(issueIds.filter(id => RUN_ID_PATTERN.test(id)))]; + if (!ids.length) return new Set(); + const result = await db.execute<{ issue_id: string }>(sql` + with recursive chain(issue_id, ancestor_id, depth) as ( + select ${issues.id}, ${issues.id}, 0 from ${issues} + where ${issues.companyId} = ${companyId} and ${issues.id} in (${sql.join(ids.map(id => sql`${id}::uuid`), sql`, `)}) + union all + select chain.issue_id, ancestor.parent_id, chain.depth + 1 + from chain + join ${issues} ancestor on ancestor.id = chain.ancestor_id + where ancestor.company_id = ${companyId} and ancestor.created_by_user_id is null + and chain.depth < ${PERSON_TREE_ANCESTOR_DEPTH} + ) + select distinct chain.issue_id from chain + join ${issues} ancestor on ancestor.id = chain.ancestor_id + where ancestor.company_id = ${companyId} and ancestor.created_by_user_id is not null`); + return new Set([...result].map(row => String(row.issue_id))); +} + /** * The facts and share counters the queued-run dispatcher orders an agent's * queued runs by. The dispatch path and the "position in line" a waiting task @@ -146,8 +168,15 @@ export async function queuedRunDispatchFacts( const sourceRunId = sourceRunIdOf(run); return Boolean(sourceRunId && personSourceRunIds.has(sourceRunId)); }; + const issueIdOf = (run: R) => readNonEmptyString(parseObject(run.contextSnapshot).issueId); + const personTreeIssueIds = personOriginChecked + ? await listPersonTreeIssueIds(db, agent.companyId, queuedRuns.flatMap(run => { + const issueId = issueIdOf(run); + return issueId && issueById.has(issueId) && !issueById.get(issueId)?.createdByUserId ? [issueId] : []; + })) + : new Set(); const facts = queuedRuns.map(run => { - const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + const issueId = issueIdOf(run); const ready = issueId ? (dependencyReadiness.get(issueId)?.isDependencyReady ?? true) : true; const issue = issueId ? issueById.get(issueId) : null; const sourceRunId = sourceRunIdOf(run); @@ -161,7 +190,7 @@ export async function queuedRunDispatchFacts( readinessRank: issueId ? (ready ? (issue?.status === "in_progress" ? 0 : 1) : 3) : 2, priorityRank: issueRunPriorityRank(issue?.priority), shareDeferred: input.shareDeferred?.(run.id) ?? false, - personTask: Boolean(issue?.createdByUserId), + personTask: Boolean(issue?.createdByUserId) || Boolean(issueId && personTreeIssueIds.has(issueId)), }; }); const since = dispatchHistoryStart(facts, now); @@ -178,10 +207,7 @@ export async function queuedRunDispatchFacts( const personStarts = new Set(startRows.length ? await listPersonOriginSourceRunIds(db, agent.companyId, startRows.map(row => row.id)) : []); const startedTaskIds = [...new Set(startRows.flatMap(row => row.issueId && RUN_ID_PATTERN.test(row.issueId) ? [row.issueId] : []))]; - const personTaskIds = new Set(startedTaskIds.length - ? (await db.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, agent.companyId), - inArray(issues.id, startedTaskIds), isNotNull(issues.createdByUserId)))).map(row => row.id) - : []); + const personTaskIds = await listPersonTreeIssueIds(db, agent.companyId, startedTaskIds); const recentStarts = startRows.map(row => ({ ...row, personOrigin: personStarts.has(row.id), personWaiting: personStarts.has(row.id) || Boolean(row.issueId && personTaskIds.has(row.issueId)) })); const { startsSinceRecovery, startsSinceOldest } = dispatchCountersFromStarts(facts, recentStarts, now);