diff --git a/DESIGN.md b/DESIGN.md index 4aaba5a24a1..afb31fde548 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -144,6 +144,12 @@ Runtime discovery must not replace the turn with a generic spinner or a A queued turn shows the same `Working` header and shimmering `Thinking` activity as a running turn, starting at message submission. Keep specific server holds, such as `Waiting for maintenance` or agent setup, visible during admission. +A run that waits for one of its agent's concurrency slots is a hold too: show +`Waiting for · N slots max · P of Q in line · Configure` (the shared +`AgentQueueWait` component) with no spinner or elapsed timer, link Configure to the +agent's run policy, and take the place in line from the server's dispatch order. +Task rows show a quiet `Waiting` instead of the pulsing `Live`, and live counts +count only started runs. This is presentation only; queue and execution authority stay unchanged. A saved message that starts this queued turn stays in history, including when it reopens an older task. This placement is not a consumption receipt. Later follow-ups remain in diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index b26e445cb6b..17f32ce6e70 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -2854,7 +2854,7 @@ export { } from "./runtime-exposure/loopback-bind.js"; export { ACCOUNT_HANDLE_MAX_LENGTH, toAccountHandle } from "./account-handle.js"; export type { ExecutionContinuationEnvelope } from "./types/execution-continuation.js"; -export type { ExecutionProjection, ExecutionReconciliation, ExecutionBlocker, TaskInterruptionPresentation } from "./types/execution-projection.js"; +export type { ExecutionAgentQueue, ExecutionProjection, ExecutionReconciliation, ExecutionBlocker, TaskInterruptionPresentation } from "./types/execution-projection.js"; export { EXECUTION_RECONCILIATION_CAUSES, requiresExecutionReconciliation } from "./types/execution-projection.js"; export * from "./types/voice-sessions.js"; diff --git a/packages/shared/src/types/execution-projection.ts b/packages/shared/src/types/execution-projection.ts index 247fd3cc8e1..7a4b8567cb1 100644 --- a/packages/shared/src/types/execution-projection.ts +++ b/packages/shared/src/types/execution-projection.ts @@ -26,6 +26,23 @@ export interface ExecutionBlocker { }; } +/** + * Where a queued, never-started run waits while its agent's concurrency limit + * is full. Position follows the order the server starts the agent's queued runs. + */ +export interface ExecutionAgentQueue { + agentId: string; + agentName: string; + /** The agent's concurrency limit (runtimeConfig.heartbeat.maxConcurrentRuns, as enforced). */ + maxConcurrentRuns: number; + /** The agent's runs that are running now. */ + runningCount: number; + /** 1-based place among the agent's queued runs; 1 starts when the next slot frees. */ + position: number; + /** All of the agent's queued runs. */ + queuedCount: number; +} + /** Presentation of existing execution records, not a second task status machine. */ export interface ExecutionProjection { /** The board can Stop this run now: any queued or running run, or a failed native run whose retry is still pending. */ @@ -53,6 +70,8 @@ export interface ExecutionProjection { permittedActions: Array<"inspect_run" | "inspect_recovery">; predecessorRunId: string | null; successorRunId: string | null; + /** Set when phase is "queued" because the agent's concurrency limit is full (cause "agent_at_capacity"). */ + agentQueue?: ExecutionAgentQueue | null; } /** These incidents require an explicit reconciliation, never a generic Retry. */ diff --git a/packages/shared/src/types/issue.ts b/packages/shared/src/types/issue.ts index 725bd6f677b..1a59939658d 100644 --- a/packages/shared/src/types/issue.ts +++ b/packages/shared/src/types/issue.ts @@ -1001,6 +1001,8 @@ export interface Issue { watchdog?: IssueWatchdogSummary | null; scheduledRetry?: IssueScheduledRetry | null; liveDescendantCount?: number; + /** Descendants whose run started; excludes runs still waiting for an agent slot. */ + runningDescendantCount?: number; relatedWork?: IssueRelatedWorkSummary; referencedIssueIdentifiers?: string[]; planDocument?: IssueDocument | null; @@ -1076,6 +1078,8 @@ export type CompactIssue = Pick< blockedInboxAttention?: IssueBlockedInboxAttention | null; scheduledRetry?: IssueScheduledRetry | null; liveDescendantCount?: number; + /** Descendants whose run started; excludes runs still waiting for an agent slot. */ + runningDescendantCount?: number; myLastTouchAt?: Date | null; lastExternalCommentAt?: Date | null; lastActivityAt?: Date | null; diff --git a/server/src/__tests__/issue-list-assignee-filter-routes.test.ts b/server/src/__tests__/issue-list-assignee-filter-routes.test.ts index 1960ccd28b6..ce3d6cd6e1b 100644 --- a/server/src/__tests__/issue-list-assignee-filter-routes.test.ts +++ b/server/src/__tests__/issue-list-assignee-filter-routes.test.ts @@ -1087,6 +1087,9 @@ describeEmbeddedPostgres("issue list routes assigneeAgentId filter", () => { expect(withSummary.body[0]).toMatchObject({ id: rootIssueId, liveDescendantCount: 1, + // The live grandchild's run is queued and never started: it covers the + // blocked parent but is not running below it. + runningDescendantCount: 0, }); }); diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index eef96b704c6..eb8be845b59 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -7262,7 +7262,9 @@ export function agentRoutes( inArray(heartbeatRuns.status, ["queued", "running"]), ), ) - .orderBy(desc(heartbeatRuns.createdAt)); + // Running runs first: a deep queue must not push the work actually + // running past the limit. + .orderBy(sql`case when ${heartbeatRuns.status} = 'running' then 0 else 1 end`, desc(heartbeatRuns.createdAt)); const liveRuns = distinctTasks ? [] : await liveRunsQuery.limit(limit); let rows = liveRuns; diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index c9340bcce9d..af797a9618d 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -3113,6 +3113,9 @@ function toCompactIssue(issue: any): CompactIssue { ...(issue.liveDescendantCount !== undefined ? { liveDescendantCount: issue.liveDescendantCount } : {}), + ...(issue.runningDescendantCount !== undefined + ? { runningDescendantCount: issue.runningDescendantCount } + : {}), ...(issue.myLastTouchAt !== undefined ? { myLastTouchAt: issue.myLastTouchAt } : {}), diff --git a/server/src/services/execution-projection-agent-queue.test.ts b/server/src/services/execution-projection-agent-queue.test.ts new file mode 100644 index 00000000000..b1e31cce202 --- /dev/null +++ b/server/src/services/execution-projection-agent-queue.test.ts @@ -0,0 +1,119 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { executionProjectionsForRuns } from "./execution-projection.js"; +import { agentQueuedRunLine, clearAgentQueueLineCache, worktreeExecutionCutoff } from "./heartbeat/queued-run-line.js"; +import { issueService } from "./issues.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const now = new Date("2026-11-13T12:00:00Z"); +const minutesAgo = (minutes: number) => new Date(now.getTime() - minutes * 60_000); + +(support.supported ? describe : describe.skip)("execution projection: waiting behind the agent's limit", () => { + let database: Awaited>; + let db: ReturnType; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("agent-queue-projection-"); + db = createDb(database.connectionString); + }, 60_000); + afterAll(async () => { await database?.cleanup(); }); + beforeEach(() => clearAgentQueueLineCache()); + + async function fixture(options: { maxConcurrentRuns: number; running: number }) { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Queue test", issuePrefix: `Q${companyId.slice(0, 8)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Codie", adapterType: "claude_local", + runtimeConfig: { heartbeat: { maxConcurrentRuns: options.maxConcurrentRuns } } }); + for (let index = 0; index < options.running; index += 1) { + await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running", + startedAt: minutesAgo(90), createdAt: minutesAgo(90) }); + } + async function queue(input: { priority: "critical" | "high" | "medium" | "low"; createdMinutesAgo: number; wakeReason?: string }) { + const issueId = randomUUID(), runId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: `Task ${input.priority}`, assigneeAgentId: agentId, + status: "in_progress", priority: input.priority }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "queued", issueId, + createdAt: minutesAgo(input.createdMinutesAgo), + contextSnapshot: { issueId, ...(input.wakeReason ? { wakeReason: input.wakeReason } : {}) } }); + return runId; + } + return { companyId, agentId, queue }; + } + + it("names the agent, its limit and each run's place in the order dispatch starts them", async () => { + const f = await fixture({ maxConcurrentRuns: 2, running: 2 }); + // Created oldest first, but dispatch starts a person's retry first, then by priority. + const lowOld = await f.queue({ priority: "low", createdMinutesAgo: 30 }); + const medium = await f.queue({ priority: "medium", createdMinutesAgo: 20 }); + const critical = await f.queue({ priority: "critical", createdMinutesAgo: 10 }); + const personRetry = await f.queue({ priority: "low", createdMinutesAgo: 5, wakeReason: "task_retry" }); + + const projections = await executionProjectionsForRuns(db, f.companyId, [lowOld, medium, critical, personRetry], now); + const queueOf = (runId: string) => projections.get(runId)?.agentQueue; + expect(projections.get(critical)).toMatchObject({ + phase: "queued", + label: "Waiting for Codie", + cause: "agent_at_capacity", + agentQueue: { agentId: f.agentId, agentName: "Codie", maxConcurrentRuns: 2, runningCount: 2, position: 2, queuedCount: 4 }, + }); + expect([personRetry, critical, medium, lowOld].map(runId => queueOf(runId)?.position)).toEqual([1, 2, 3, 4]); + + // The position is the dispatch line itself, not a client-side sort. + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + expect(await agentQueuedRunLine(db, issueService(db), agent!, now)).toEqual([personRetry, critical, medium, lowOld]); + }); + + it("does not claim a run waits on the limit while the agent has a free slot", async () => { + const f = await fixture({ maxConcurrentRuns: 3, running: 2 }); + const runId = await f.queue({ priority: "high", createdMinutesAgo: 5 }); + const projection = (await executionProjectionsForRuns(db, f.companyId, [runId], now)).get(runId); + expect(projection).toMatchObject({ phase: "queued", label: "Queued", cause: null }); + expect(projection?.agentQueue).toBeUndefined(); + + // Once the limit is full the same run explains the wait. + await db.update(agents).set({ runtimeConfig: { heartbeat: { maxConcurrentRuns: 2 } } }).where(eq(agents.id, f.agentId)); + expect((await executionProjectionsForRuns(db, f.companyId, [runId], now)).get(runId)).toMatchObject({ + label: "Waiting for Codie", + agentQueue: { maxConcurrentRuns: 2, runningCount: 2, position: 1, queuedCount: 1 }, + }); + }); + + it("leaves out queued runs that a preview worktree's cutoff keeps from starting", async () => { + const f = await fixture({ maxConcurrentRuns: 1, running: 1 }); + const copiedOld = await f.queue({ priority: "critical", createdMinutesAgo: 120 }); + const fresh = await f.queue({ priority: "low", createdMinutesAgo: 5 }); + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + expect(await agentQueuedRunLine(db, issueService(db), agent!, now, { cutoff: null })).toEqual([copiedOld, fresh]); + expect(await agentQueuedRunLine(db, issueService(db), agent!, now, { cutoff: minutesAgo(60) })).toEqual([fresh]); + // Outside a worktree there is no cutoff and no settings read. + expect(await worktreeExecutionCutoff(db, {})).toBeNull(); + }); + + it("keeps a deep queue's place in line to a few reads, not one per run", async () => { + const f = await fixture({ maxConcurrentRuns: 20, running: 20 }); + const runIds: string[] = []; + for (let index = 0; index < 300; index += 1) { + runIds.push(await f.queue({ priority: (["critical", "high", "medium", "low"] as const)[index % 4], createdMinutesAgo: 300 - index, + ...(index % 9 === 0 ? { wakeReason: "issue_continuation_needed" } : {}) })); + } + const started = performance.now(); + const projections = await executionProjectionsForRuns(db, f.companyId, runIds, now); + const elapsedMs = performance.now() - started; + const positions = runIds.map(runId => projections.get(runId)?.agentQueue?.position).sort((a, b) => a! - b!); + expect(positions).toEqual(runIds.map((_, index) => index + 1)); + expect(projections.get(runIds[0]!)?.agentQueue?.queuedCount).toBe(300); + // Generous bound for slow CI; the ordering itself is O(n log n). + expect(elapsedMs).toBeLessThan(5_000); + }); + + it("leaves running runs and other companies' runs out of the line", async () => { + const f = await fixture({ maxConcurrentRuns: 1, running: 1 }); + const mine = await f.queue({ priority: "medium", createdMinutesAgo: 5 }); + const other = await fixture({ maxConcurrentRuns: 1, running: 1 }); + await other.queue({ priority: "critical", createdMinutesAgo: 50 }); + const projection = (await executionProjectionsForRuns(db, f.companyId, [mine], now)).get(mine); + expect(projection?.agentQueue).toMatchObject({ position: 1, queuedCount: 1, runningCount: 1 }); + }); +}); diff --git a/server/src/services/execution-projection.ts b/server/src/services/execution-projection.ts index 06a351c7162..714cc9546d8 100644 --- a/server/src/services/execution-projection.ts +++ b/server/src/services/execution-projection.ts @@ -22,6 +22,7 @@ import { getStartupRecoveryState, startupQueuedRunPassOutstanding } from "../sta import { hasPendingRunCleanup } from "./execution-blocker.js"; import { isAgentAwaitingSetup } from "../modules/agent-lifecycle/index.js"; import { NATIVE_RESPONSE_WAKE_WAITING_REASON } from "./native-runtime/status-arbiter.js"; +import { agentQueueLinesForRuns } from "./heartbeat/queued-run-line.js"; const MAX_PREDECESSOR_CLEANUP_CHECKS = 16; const text = (v: unknown) => (typeof v === "string" ? v : null); const executionRunColumns = { @@ -203,6 +204,11 @@ export async function executionProjectionsForRuns( retryRequired: sql`coalesce(${agents.lifecycleOperation}->>'retryRequired', 'false') = 'true'` }) .from(agents).where(and(eq(agents.companyId, companyId), inArray(agents.id, setupAgentIds)))) : []; const setupByAgent = new Map(setupAgents.map(agent => [agent.id, agent])); + // Only never-started runs wait on the agent's limit; their place in line is + // read once per full agent, not once per run. + const agentQueueByRun = queuedRuns.some(run => !run.startedAt) + ? await read(() => agentQueueLinesForRuns(db, companyId, queuedRuns.filter(run => !run.startedAt), now)) + : new Map>(); const issueIdOf = (run: Run) => run.nativeIssueId ?? text(run.contextSnapshot?.issueId); const queuedIssueIds = [...new Set(queuedRuns.map(issueIdOf).filter((id): id is string => !!id))]; if (queuedIssueIds.length) { @@ -278,6 +284,7 @@ export async function executionProjectionsForRuns( settlingPredecessorKeys.has(run.id), setupByAgent.get(run.agentId), currentResponseWait(run), + agentQueueByRun.get(run.id), ), ); } @@ -305,6 +312,7 @@ export function projectExecution( waitingForPredecessor = false, setupAgent?: { lifecycleState: string; lifecycleHolds: readonly string[]; retryRequired?: boolean }, responseWaitCurrent = false, + agentQueue?: ExecutionProjection["agentQueue"], ): ExecutionProjection { const detail = coordinator?.failureDetail ?? {}; const successorRunId = text(detail.successorRunId); @@ -553,5 +561,11 @@ export function projectExecution( projection.nextAction = "The previous run on this task is still releasing its runner. This turn starts automatically when it settles."; return set("queued", "Waiting for previous run to stop"); } + if (run.status === "queued" && !run.startedAt && agentQueue) { + projection.cause = "agent_at_capacity"; + projection.agentQueue = agentQueue; + projection.nextAction = `${agentQueue.agentName} is running ${agentQueue.runningCount} of ${agentQueue.maxConcurrentRuns} tasks at once. This task is ${agentQueue.position} of ${agentQueue.queuedCount} in line and starts automatically when a slot frees.`; + return set("queued", `Waiting for ${agentQueue.agentName}`); + } return projection; } diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index cd8aee1e94f..d3f1d08b284 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -12,8 +12,8 @@ import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, reche import { deferredWorkIntentWait } from "../work-intent-outbox.js"; import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; import { createHeartbeatTimerPolicy } from "./timer-policy.js"; -import { OLDEST_DISPATCH_SHARE, PERSON_WAKE_REASONS, QUEUED_RUN_AGING_MS, RECOVERY_DISPATCH_SHARE, createQueuedRunDispatcher, - hasQueuedRunAged, isRecoveryWakeReason, isSystemRecoveryRun, oldestReadyQueuedRun } from "./queued-run-dispatch-order.js"; +import { QUEUED_RUN_AGING_MS, createQueuedRunDispatcher, hasQueuedRunAged } from "./queued-run-dispatch-order.js"; +import { queuedRunDispatchFacts } from "./queued-run-line.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js"; import { assignmentWakePayloadSql, isAssignmentWakePayload, notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; @@ -375,8 +375,6 @@ export function shouldQueueFollowupForRunningIssueWake(input: { /** Agent Chat outbox receipts. The outbox, not a skipped receipt, owns a capped message. */ const CONVERSATION_COMMENT_KEY_PREFIX = "conversation-comment:"; -const RUN_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; -const PERSON_ORIGIN_CHAIN_DEPTH = 8; function dailyLimitConversationRefusal(cap: { reason: string; observed: number; limit: number }) { // No receipt is written, so the person's message stays owed in the outbox // until the limit resets (agent-conversations.ts conversationDeliveryHold). @@ -597,21 +595,6 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const claimDueTimerHeartbeat = (agent: typeof agents.$inferSelect, now: Date, intervalSec: number) => claimDueTimerHeartbeatForAgent(db, agent, now, intervalSec); - function issueRunPriorityRank(priority: string | null | undefined) { - switch (priority) { - case "critical": - return 0; - case "high": - return 1; - case "medium": - return 2; - case "low": - return 3; - default: - return 4; - } - } - async function listQueuedRunDependencyReadiness( companyId: string, queuedRuns: Array, @@ -2368,31 +2351,6 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } } - async function listPersonOriginSourceRunIds(companyId: string, sourceRunIds: string[]) { - const uuidPattern = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; - const result = await db.execute<{ root_id: string }>(sql` - with recursive chain(root_id, run_id, depth) as ( - select source.id, source.id, 0 from ${heartbeatRuns} source - where source.company_id = ${companyId} and source.id in (${sql.join(sourceRunIds.map(id => sql`${id}::uuid`), sql`, `)}) - union all - select chain.root_id, parent.id, chain.depth + 1 - from chain - join ${heartbeatRuns} child on child.id = chain.run_id - join ${heartbeatRuns} parent on parent.company_id = ${companyId} and parent.id = coalesce( - case when child.context_snapshot->>'previousRunId' ~* ${uuidPattern} - then (child.context_snapshot->>'previousRunId')::uuid end, - child.retry_of_run_id) - where chain.depth < ${PERSON_ORIGIN_CHAIN_DEPTH} - ) - select distinct chain.root_id from chain - join ${heartbeatRuns} run on run.id = chain.run_id - where run.context_snapshot->>'wakeReason' in (${sql.join([...PERSON_WAKE_REASONS].map(reason => sql`${reason}`), sql`, `)}) - or run.context_snapshot->>'taskRetryRequestId' is not null - or exists (select 1 from ${agentWakeupRequests} wake where wake.company_id = ${companyId} - and wake.requested_by_actor_type = 'user' and (wake.run_id = run.id or wake.id = run.wakeup_request_id))`); - return [...result].map(row => String(row.root_id)); - } - async function queuedRunDispatcherForAgent( agent: typeof agents.$inferSelect, queuedRuns: Array, @@ -2401,93 +2359,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende ) { const now = new Date(); pruneDeclinedAgedClaims(now); - const wakeReasonOf = (run: typeof heartbeatRuns.$inferSelect) => - readNonEmptyString(parseObject(run.contextSnapshot).wakeReason); - const requestIds = [...new Set(queuedRuns.flatMap(run => run.wakeupRequestId ? [run.wakeupRequestId] : []))]; - const userRequests = queuedRuns.length > 1 && requestIds.length - ? await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( - eq(agentWakeupRequests.companyId, agent.companyId), inArray(agentWakeupRequests.id, requestIds), - eq(agentWakeupRequests.requestedByActorType, "user"))) - : []; - const userRequestIds = new Set(userRequests.map(row => row.id)); - const joinedUserRequests = queuedRuns.length > 1 - ? await db.select({ runId: agentWakeupRequests.runId }).from(agentWakeupRequests).where(and( - eq(agentWakeupRequests.companyId, agent.companyId), inArray(agentWakeupRequests.runId, queuedRuns.map(run => run.id)), - eq(agentWakeupRequests.agentId, agent.id), eq(agentWakeupRequests.requestedByActorType, "user"))) - : []; - const userRunIds = new Set(joinedUserRequests.flatMap(row => row.runId ? [row.runId] : [])); - const joinedByUser = (run: typeof heartbeatRuns.$inferSelect) => - userRunIds.has(run.id) || Boolean(run.wakeupRequestId && userRequestIds.has(run.wakeupRequestId)); - const sourceRunIdOf = (run: typeof heartbeatRuns.$inferSelect) => - readNonEmptyString(parseObject(run.contextSnapshot).previousRunId) ?? run.retryOfRunId ?? null; - const sourceRunIds = queuedRuns.length > 1 - ? [...new Set(queuedRuns.flatMap(run => { - const sourceRunId = joinedByUser(run) ? null : sourceRunIdOf(run); - return sourceRunId && RUN_ID_PATTERN.test(sourceRunId) ? [sourceRunId] : []; - }))] - : []; - const sourceRows = sourceRunIds.length - ? await db.select({ id: heartbeatRuns.id, finishedAt: heartbeatRuns.finishedAt }).from(heartbeatRuns) - .where(and(eq(heartbeatRuns.companyId, agent.companyId), inArray(heartbeatRuns.id, sourceRunIds))) - : []; - const sourceEndedAt = new Map(sourceRows.map(row => [row.id, row.finishedAt])); - const personSourceRunIds = new Set(sourceRows.length - ? await listPersonOriginSourceRunIds(agent.companyId, sourceRows.map(row => row.id)) - : []); - const requestedByUser = (run: typeof heartbeatRuns.$inferSelect) => { - if (joinedByUser(run)) return true; - const sourceRunId = sourceRunIdOf(run); - return Boolean(sourceRunId && personSourceRunIds.has(sourceRunId)); - }; - const facts = queuedRuns.map(run => { - const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); - const ready = issueId ? (dependencyReadiness.get(issueId)?.isDependencyReady ?? true) : true; - const issue = issueId ? issueById.get(issueId) : null; - const sourceRunId = sourceRunIdOf(run); - return { - run, - id: run.id, - createdAt: run.createdAt, - wakeReason: wakeReasonOf(run), - requestedByUser: requestedByUser(run), - sourceEndedAt: sourceRunId ? sourceEndedAt.get(sourceRunId) ?? null : null, - readinessRank: issueId ? (ready ? (issue?.status === "in_progress" ? 0 : 1) : 3) : 2, - priorityRank: issueRunPriorityRank(issue?.priority), - shareDeferred: declinedAgedClaimAt.has(run.id), - }; - }); - const recoveryRuns = facts.filter(entry => isSystemRecoveryRun(entry)); - const oldestReady = oldestReadyQueuedRun(facts); - const agedOldest = oldestReady && hasQueuedRunAged(oldestReady, now) ? oldestReady : null; - const countsRecovery = recoveryRuns.length > 0 && recoveryRuns.length < facts.length; - let startsSinceRecovery = 0; - let startsSinceOldest = 0; - if (countsRecovery || (agedOldest && facts.length > 1)) { - const oldestRecoveryAt = countsRecovery ? Math.min(...recoveryRuns.map(entry => entry.createdAt.getTime())) : Infinity; - const since = new Date(Math.min(oldestRecoveryAt, agedOldest?.createdAt.getTime() ?? Infinity)); - const recentStarts = await db.select({ id: heartbeatRuns.id, wakeReason: sql`${heartbeatRuns.contextSnapshot}->>'wakeReason'`, - startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt }) - .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), - isNotNull(heartbeatRuns.startedAt), gte(heartbeatRuns.startedAt, since))) - .orderBy(desc(heartbeatRuns.startedAt)) - .limit(Math.max(RECOVERY_DISPATCH_SHARE, OLDEST_DISPATCH_SHARE)); - const startsSince = (at: number) => recentStarts.filter(row => row.startedAt && row.startedAt.getTime() >= at); - if (countsRecovery) { - const starts = startsSince(oldestRecoveryAt).slice(0, RECOVERY_DISPATCH_SHARE); - const recoveryStartIds = starts.flatMap(row => isRecoveryWakeReason(row.wakeReason) ? [row.id] : []); - const personStarts = new Set(recoveryStartIds.length - ? await listPersonOriginSourceRunIds(agent.companyId, recoveryStartIds) : []); - const lastRecovery = starts.findIndex(row => isRecoveryWakeReason(row.wakeReason) && !personStarts.has(row.id)); - startsSinceRecovery = lastRecovery < 0 ? starts.length : lastRecovery; - } - if (agedOldest && facts.length > 1) { - const starts = startsSince(agedOldest.createdAt.getTime()).slice(0, OLDEST_DISPATCH_SHARE); - const lastOldest = starts.findIndex(row => row.createdAt.getTime() <= agedOldest.createdAt.getTime()); - startsSinceOldest = lastOldest < 0 ? starts.length : lastOldest; - } - } - const dispatcher = createQueuedRunDispatcher(facts, { now, startsSinceRecovery, startsSinceOldest }); + const { facts, dispatchInput } = await queuedRunDispatchFacts(db, { + agent, queuedRuns, dependencyReadiness, issueById, now, shareDeferred: runId => declinedAgedClaimAt.has(runId) }); + const dispatcher = createQueuedRunDispatcher(facts, dispatchInput); const byRun = new Map(facts.map(entry => [entry.run, entry])); return { next: () => dispatcher.next()?.run ?? null, diff --git a/server/src/services/heartbeat/queued-run-dispatch-order.ts b/server/src/services/heartbeat/queued-run-dispatch-order.ts index 6354b385c80..149d4be515b 100644 --- a/server/src/services/heartbeat/queued-run-dispatch-order.ts +++ b/server/src/services/heartbeat/queued-run-dispatch-order.ts @@ -66,20 +66,24 @@ export function oldestReadyQueuedRun(runs: rea return oldest; } -export function createQueuedRunDispatcher( - runs: readonly T[], - input: QueuedRunDispatchInput, -) { +function sortForDispatch(runs: readonly T[], now: Date): T[] { const tier = (run: T) => { if (run.readinessRank >= NOT_READY_RANK) return 3; - const classTier = CLASS_TIER[queuedRunDispatchClass(run, input.now)]; - return hasQueuedRunAged(run, input.now) ? Math.min(classTier, CLASS_TIER.work) : classTier; + const classTier = CLASS_TIER[queuedRunDispatchClass(run, now)]; + return hasQueuedRunAged(run, now) ? Math.min(classTier, CLASS_TIER.work) : classTier; }; - const remaining = [...runs].sort((left, right) => + return [...runs].sort((left, right) => tier(left) - tier(right) || left.readinessRank - right.readinessRank || left.priorityRank - right.priorityRank || left.createdAt.getTime() - right.createdAt.getTime()); +} + +export function createQueuedRunDispatcher( + runs: readonly T[], + input: QueuedRunDispatchInput, +) { + const remaining = sortForDispatch(runs, input.now); let sinceRecovery = input.startsSinceRecovery; let sinceOldest = input.startsSinceOldest ?? 0; return { @@ -112,3 +116,62 @@ export function orderQueuedRunsForDispatch( } return result; } + +/** + * The whole dispatch order in O(n log n), for showing each queued run's place + * in line. It replays exactly what {@link createQueuedRunDispatcher} does when + * every offered run is claimed ({@link orderQueuedRunsForDispatch}), without + * that replay's rescans of the remaining queue after each step. Dispatch itself + * stops after the free slots, so it keeps the simpler dispatcher. + */ +export function queuedRunDispatchLine( + runs: readonly T[], + input: QueuedRunDispatchInput, +): T[] { + const order = sortForDispatch(runs, input.now); + const removed = new Uint8Array(order.length); + // Eligible for the oldest share, oldest first; ties keep dispatch order, + // as oldestReadyQueuedRun keeps the first of equal ages. + const byAge = order.map((_, index) => index) + .filter(index => order[index]!.readinessRank < NOT_READY_RANK && !order[index]!.shareDeferred) + .sort((left, right) => order[left]!.createdAt.getTime() - order[right]!.createdAt.getTime() || left - right); + const recovery = order.map((_, index) => index) + .filter(index => order[index]!.readinessRank < NOT_READY_RANK && isSystemRecoveryRun(order[index]!)); + let head = 0; + let agePointer = 0; + let recoveryPointer = 0; + const firstOf = (indexes: number[] | null, pointer: number) => { + if (!indexes) { + while (head < order.length && removed[head]) head += 1; + return head < order.length ? head : -1; + } + while (pointer < indexes.length && removed[indexes[pointer]!]) pointer += 1; + return pointer; + }; + const oldestIndex = () => { + agePointer = firstOf(byAge, agePointer); + return agePointer < byAge.length ? byAge[agePointer]! : -1; + }; + let sinceRecovery = input.startsSinceRecovery; + let sinceOldest = input.startsSinceOldest ?? 0; + const result: T[] = []; + while (result.length < order.length) { + let pick = -1; + if (sinceOldest >= OLDEST_DISPATCH_SHARE - 1) { + const oldest = oldestIndex(); + if (oldest >= 0 && hasQueuedRunAged(order[oldest]!, input.now)) pick = oldest; + } + if (pick < 0 && sinceRecovery >= RECOVERY_DISPATCH_SHARE - 1) { + recoveryPointer = firstOf(recovery, recoveryPointer); + if (recoveryPointer < recovery.length) pick = recovery[recoveryPointer]!; + } + if (pick < 0) pick = firstOf(null, 0); + removed[pick] = 1; + const run = order[pick]!; + result.push(run); + sinceRecovery = isSystemRecoveryRun(run) ? 0 : sinceRecovery + 1; + const oldest = oldestIndex(); + sinceOldest = oldest < 0 || order[oldest]!.createdAt.getTime() >= run.createdAt.getTime() ? 0 : sinceOldest + 1; + } + return result; +} diff --git a/server/src/services/heartbeat/queued-run-line.test.ts b/server/src/services/heartbeat/queued-run-line.test.ts new file mode 100644 index 00000000000..ded2a0c0d8f --- /dev/null +++ b/server/src/services/heartbeat/queued-run-line.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { + OLDEST_DISPATCH_SHARE, + QUEUED_RUN_AGING_MS, + RECOVERY_DISPATCH_SHARE, + orderQueuedRunsForDispatch, + queuedRunDispatchLine, + type QueuedRunDispatchFacts, +} from "./queued-run-dispatch-order.js"; + +const now = new Date("2026-11-13T12:00:00Z"); +const reasons = [null, null, "issue_continuation_needed", "issue_recovery_action_restored", "task_retry", "issue_assigned"]; + +// Small deterministic PRNG so failures reproduce. +function random(seed: number) { + let state = seed >>> 0; + return () => { + state = (state * 1664525 + 1013904223) >>> 0; + return state / 2 ** 32; + }; +} + +function queue(seed: number, size: number): QueuedRunDispatchFacts[] { + const next = random(seed); + const pick = (values: readonly T[]) => values[Math.floor(next() * values.length)]!; + return Array.from({ length: size }, (_, index) => { + // Coarse ages so equal creation times and the aging boundary both occur. + const createdAt = new Date(now.getTime() - Math.floor(next() * 12) * (QUEUED_RUN_AGING_MS / 4)); + return { + id: `run-${index}`, + createdAt, + wakeReason: pick(reasons), + requestedByUser: next() < 0.15, + sourceEndedAt: next() < 0.5 ? new Date(createdAt.getTime() - Math.floor(next() * 4) * 20 * 60_000) : null, + readinessRank: pick([0, 0, 1, 2, 3]), + priorityRank: pick([0, 1, 2, 3, 4]), + shareDeferred: next() < 0.1, + }; + }); +} + +describe("queuedRunDispatchLine", () => { + it("matches the dispatcher's own claim-every-run order", () => { + for (let seed = 1; seed <= 400; seed += 1) { + const runs = queue(seed, 1 + (seed % 60)); + const input = { + now, + startsSinceRecovery: seed % (RECOVERY_DISPATCH_SHARE + 1), + startsSinceOldest: seed % (OLDEST_DISPATCH_SHARE + 1), + }; + expect(queuedRunDispatchLine(runs, input).map(run => run.id), `seed ${seed}`) + .toEqual(orderQueuedRunsForDispatch(runs, input).map(run => run.id)); + } + }); + + it("orders a 10,000-run queue without rescanning it per step", () => { + const runs = queue(7, 10_000); + const input = { now, startsSinceRecovery: 0, startsSinceOldest: 0 }; + const started = performance.now(); + const line = queuedRunDispatchLine(runs, input); + const elapsedMs = performance.now() - started; + expect(line).toHaveLength(10_000); + expect(new Set(line.map(run => run.id)).size).toBe(10_000); + // The quadratic replay takes about 600 ms here; the line takes a few ms. + expect(elapsedMs).toBeLessThan(250); + }); +}); diff --git a/server/src/services/heartbeat/queued-run-line.ts b/server/src/services/heartbeat/queued-run-line.ts new file mode 100644 index 00000000000..097cf00fbd9 --- /dev/null +++ b/server/src/services/heartbeat/queued-run-line.ts @@ -0,0 +1,316 @@ +import { and, asc, desc, eq, gte, inArray, isNotNull, sql } from "drizzle-orm"; +import { agentWakeupRequests, agents, heartbeatRuns, issues, type Db } from "@paperclipai/db"; +import type { ExecutionAgentQueue } from "@paperclipai/shared"; +import { parseObject } from "../../adapters/utils.js"; +import { + OLDEST_DISPATCH_SHARE, + PERSON_WAKE_REASONS, + RECOVERY_DISPATCH_SHARE, + hasQueuedRunAged, + isRecoveryWakeReason, + isSystemRecoveryRun, + oldestReadyQueuedRun, + queuedRunDispatchLine, + type QueuedRunDispatchFacts, + type QueuedRunDispatchInput, +} from "./queued-run-dispatch-order.js"; +import { parseHeartbeatPolicy } from "./timer-policy.js"; +import { instanceSettingsService, isTruthyRuntimeEnvValue, resolveWorktreeRunExecutionActivation } from "../instance-settings.js"; + +type HeartbeatRun = typeof heartbeatRuns.$inferSelect; +type Agent = typeof agents.$inferSelect; + +const RUN_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const PERSON_ORIGIN_CHAIN_DEPTH = 8; + +/** The run fields dispatch ordering reads. */ +export type QueuedRunDispatchRun = Pick; + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +export function issueRunPriorityRank(priority: string | null | undefined) { + switch (priority) { + case "critical": return 0; + case "high": return 1; + case "medium": return 2; + case "low": return 3; + default: return 4; + } +} + +async function listPersonOriginSourceRunIds(db: Db, companyId: string, sourceRunIds: string[]) { + const uuidPattern = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; + const result = await db.execute<{ root_id: string }>(sql` + with recursive chain(root_id, run_id, depth) as ( + select source.id, source.id, 0 from ${heartbeatRuns} source + where source.company_id = ${companyId} and source.id in (${sql.join(sourceRunIds.map(id => sql`${id}::uuid`), sql`, `)}) + union all + select chain.root_id, parent.id, chain.depth + 1 + from chain + join ${heartbeatRuns} child on child.id = chain.run_id + join ${heartbeatRuns} parent on parent.company_id = ${companyId} and parent.id = coalesce( + case when child.context_snapshot->>'previousRunId' ~* ${uuidPattern} + then (child.context_snapshot->>'previousRunId')::uuid end, + child.retry_of_run_id) + where chain.depth < ${PERSON_ORIGIN_CHAIN_DEPTH} + ) + select distinct chain.root_id from chain + join ${heartbeatRuns} run on run.id = chain.run_id + where run.context_snapshot->>'wakeReason' in (${sql.join([...PERSON_WAKE_REASONS].map(reason => sql`${reason}`), sql`, `)}) + or run.context_snapshot->>'taskRetryRequestId' is not null + or exists (select 1 from ${agentWakeupRequests} wake where wake.company_id = ${companyId} + and wake.requested_by_actor_type = 'user' and (wake.run_id = run.id or wake.id = run.wakeup_request_id))`); + return [...result].map(row => String(row.root_id)); +} + +/** + * The facts and share counters the queued-run dispatcher orders an agent's + * queued runs by. The dispatch path and the "position in line" a waiting task + * shows both read this one builder, so the line the board sees is the order + * the server starts runs in. + */ +export async function queuedRunDispatchFacts( + db: Db, + input: { + agent: Pick; + queuedRuns: R[]; + dependencyReadiness: Map; + issueById: Map; + shareDeferred?: (runId: string) => boolean; + now: Date; + }, +): Promise<{ facts: Array; dispatchInput: QueuedRunDispatchInput }> { + const { agent, queuedRuns, dependencyReadiness, issueById, now } = input; + const wakeReasonOf = (run: R) => readNonEmptyString(parseObject(run.contextSnapshot).wakeReason); + const requestIds = [...new Set(queuedRuns.flatMap(run => run.wakeupRequestId ? [run.wakeupRequestId] : []))]; + const userRequests = queuedRuns.length > 1 && requestIds.length + ? await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, agent.companyId), inArray(agentWakeupRequests.id, requestIds), + eq(agentWakeupRequests.requestedByActorType, "user"))) + : []; + const userRequestIds = new Set(userRequests.map(row => row.id)); + const joinedUserRequests = queuedRuns.length > 1 + ? await db.select({ runId: agentWakeupRequests.runId }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, agent.companyId), inArray(agentWakeupRequests.runId, queuedRuns.map(run => run.id)), + eq(agentWakeupRequests.agentId, agent.id), eq(agentWakeupRequests.requestedByActorType, "user"))) + : []; + const userRunIds = new Set(joinedUserRequests.flatMap(row => row.runId ? [row.runId] : [])); + const joinedByUser = (run: R) => + userRunIds.has(run.id) || Boolean(run.wakeupRequestId && userRequestIds.has(run.wakeupRequestId)); + const sourceRunIdOf = (run: R) => + readNonEmptyString(parseObject(run.contextSnapshot).previousRunId) ?? run.retryOfRunId ?? null; + const sourceRunIds = queuedRuns.length > 1 + ? [...new Set(queuedRuns.flatMap(run => { + const sourceRunId = joinedByUser(run) ? null : sourceRunIdOf(run); + return sourceRunId && RUN_ID_PATTERN.test(sourceRunId) ? [sourceRunId] : []; + }))] + : []; + const sourceRows = sourceRunIds.length + ? await db.select({ id: heartbeatRuns.id, finishedAt: heartbeatRuns.finishedAt }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, agent.companyId), inArray(heartbeatRuns.id, sourceRunIds))) + : []; + const sourceEndedAt = new Map(sourceRows.map(row => [row.id, row.finishedAt])); + const personSourceRunIds = new Set(sourceRows.length + ? await listPersonOriginSourceRunIds(db, agent.companyId, sourceRows.map(row => row.id)) + : []); + const requestedByUser = (run: R) => { + if (joinedByUser(run)) return true; + const sourceRunId = sourceRunIdOf(run); + return Boolean(sourceRunId && personSourceRunIds.has(sourceRunId)); + }; + const facts = queuedRuns.map(run => { + const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + const ready = issueId ? (dependencyReadiness.get(issueId)?.isDependencyReady ?? true) : true; + const issue = issueId ? issueById.get(issueId) : null; + const sourceRunId = sourceRunIdOf(run); + return { + run, + id: run.id, + createdAt: run.createdAt, + wakeReason: wakeReasonOf(run), + requestedByUser: requestedByUser(run), + sourceEndedAt: sourceRunId ? sourceEndedAt.get(sourceRunId) ?? null : null, + readinessRank: issueId ? (ready ? (issue?.status === "in_progress" ? 0 : 1) : 3) : 2, + priorityRank: issueRunPriorityRank(issue?.priority), + shareDeferred: input.shareDeferred?.(run.id) ?? false, + }; + }); + const recoveryRuns = facts.filter(entry => isSystemRecoveryRun(entry)); + const oldestReady = oldestReadyQueuedRun(facts); + const agedOldest = oldestReady && hasQueuedRunAged(oldestReady, now) ? oldestReady : null; + const countsRecovery = recoveryRuns.length > 0 && recoveryRuns.length < facts.length; + let startsSinceRecovery = 0; + let startsSinceOldest = 0; + if (countsRecovery || (agedOldest && facts.length > 1)) { + const oldestRecoveryAt = countsRecovery ? Math.min(...recoveryRuns.map(entry => entry.createdAt.getTime())) : Infinity; + const since = new Date(Math.min(oldestRecoveryAt, agedOldest?.createdAt.getTime() ?? Infinity)); + const recentStarts = await db.select({ id: heartbeatRuns.id, wakeReason: sql`${heartbeatRuns.contextSnapshot}->>'wakeReason'`, + startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt }) + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), + isNotNull(heartbeatRuns.startedAt), gte(heartbeatRuns.startedAt, since))) + .orderBy(desc(heartbeatRuns.startedAt)) + .limit(Math.max(RECOVERY_DISPATCH_SHARE, OLDEST_DISPATCH_SHARE)); + const startsSince = (at: number) => recentStarts.filter(row => row.startedAt && row.startedAt.getTime() >= at); + if (countsRecovery) { + const starts = startsSince(oldestRecoveryAt).slice(0, RECOVERY_DISPATCH_SHARE); + const recoveryStartIds = starts.flatMap(row => isRecoveryWakeReason(row.wakeReason) ? [row.id] : []); + const personStarts = new Set(recoveryStartIds.length + ? await listPersonOriginSourceRunIds(db, agent.companyId, recoveryStartIds) : []); + const lastRecovery = starts.findIndex(row => isRecoveryWakeReason(row.wakeReason) && !personStarts.has(row.id)); + startsSinceRecovery = lastRecovery < 0 ? starts.length : lastRecovery; + } + if (agedOldest && facts.length > 1) { + const starts = startsSince(agedOldest.createdAt.getTime()).slice(0, OLDEST_DISPATCH_SHARE); + const lastOldest = starts.findIndex(row => row.createdAt.getTime() <= agedOldest.createdAt.getTime()); + startsSinceOldest = lastOldest < 0 ? starts.length : lastOldest; + } + } + return { facts, dispatchInput: { now, startsSinceRecovery, startsSinceOldest } }; +} + +/** At most this many agents' lines are computed for one projection read. */ +export const MAX_AGENT_QUEUE_LINES_PER_READ = 8; + +/** + * Where each queued, never-started run waits in its agent's line, for agents + * whose concurrency limit is full. An agent with a free slot is omitted: its + * runs are not waiting on the limit. One read of the agents and their running + * counts, then one line per full agent. + */ +export async function agentQueueLinesForRuns( + db: Db, + companyId: string, + queuedRuns: Array<{ id: string; agentId: string }>, + now = new Date(), +): Promise> { + const result = new Map(); + const agentIds = [...new Set(queuedRuns.map(run => run.agentId))]; + if (!agentIds.length) return result; + const [agentRows, runningRows] = await Promise.all([ + db.select().from(agents).where(and(eq(agents.companyId, companyId), inArray(agents.id, agentIds))), + db.select({ agentId: heartbeatRuns.agentId, count: sql`count(*)::int` }) + .from(heartbeatRuns) + .where(and(inArray(heartbeatRuns.agentId, agentIds), eq(heartbeatRuns.status, "running"))) + .groupBy(heartbeatRuns.agentId), + ]); + const runningByAgent = new Map(runningRows.map(row => [row.agentId, Number(row.count)])); + const fullAgents = agentRows.flatMap(agent => { + const maxConcurrentRuns = parseHeartbeatPolicy(agent).maxConcurrentRuns; + const runningCount = runningByAgent.get(agent.id) ?? 0; + return runningCount >= maxConcurrentRuns ? [{ agent, maxConcurrentRuns, runningCount }] : []; + }).slice(0, MAX_AGENT_QUEUE_LINES_PER_READ); + if (!fullAgents.length) return result; + // Lazy: the issue service imports the execution projection that calls this. + const { issueService } = await import("../issues.js"); + const issuesSvc = issueService(db); + const wanted = new Set(queuedRuns.map(run => run.id)); + await Promise.all(fullAgents.map(async ({ agent, maxConcurrentRuns, runningCount }) => { + const line = await cachedAgentQueuedRunLine(db, issuesSvc, agent, now); + line.forEach((runId, index) => { + if (!wanted.has(runId)) return; + result.set(runId, { + agentId: agent.id, + agentName: agent.name, + maxConcurrentRuns, + runningCount, + position: index + 1, + queuedCount: line.length, + }); + }); + })); + return result; +} + +/** The agent's queued runs, in the order dispatch starts them (the same set and order as startNextQueuedRunForAgent). */ +export async function agentQueuedRunLine( + db: Db, + issuesSvc: { listDependencyReadiness: (companyId: string, issueIds: string[]) => Promise> }, + agent: Agent, + now = new Date(), + options: { cutoff?: Date | null } = {}, +): Promise { + const cutoff = options.cutoff !== undefined ? options.cutoff : await worktreeExecutionCutoff(db); + const queuedRuns = await db.select({ + id: heartbeatRuns.id, + createdAt: heartbeatRuns.createdAt, + wakeupRequestId: heartbeatRuns.wakeupRequestId, + retryOfRunId: heartbeatRuns.retryOfRunId, + // Only the keys ordering reads; full snapshots are large. + contextSnapshot: sql>`jsonb_build_object( + 'issueId', ${heartbeatRuns.contextSnapshot}->'issueId', + 'wakeReason', ${heartbeatRuns.contextSnapshot}->'wakeReason', + 'previousRunId', ${heartbeatRuns.contextSnapshot}->'previousRunId')`, + }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), + eq(heartbeatRuns.status, "queued"), cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined)) + .orderBy(asc(heartbeatRuns.createdAt)); + if (!queuedRuns.length) return []; + const issueIds = [...new Set(queuedRuns.flatMap(run => { + const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + return issueId ? [issueId] : []; + }))]; + const [dependencyReadiness, issueRows] = issueIds.length + ? await Promise.all([ + issuesSvc.listDependencyReadiness(agent.companyId, issueIds), + db.select({ id: issues.id, status: issues.status, priority: issues.priority }).from(issues) + .where(and(eq(issues.companyId, agent.companyId), inArray(issues.id, issueIds))), + ]) + : [new Map(), []]; + const { facts, dispatchInput } = await queuedRunDispatchFacts(db, { + agent, + queuedRuns, + dependencyReadiness, + issueById: new Map(issueRows.map(row => [row.id, row])), + now, + }); + return queuedRunDispatchLine(facts, dispatchInput).map(entry => entry.id); +} + +/** + * Dispatch skips queued runs created before an enabled preview worktree's + * cutoff, so the line skips them too. Outside a worktree this reads nothing. + */ +export async function worktreeExecutionCutoff(db: Db, env: NodeJS.ProcessEnv = process.env): Promise { + if (!isTruthyRuntimeEnvValue(env.PAPERCLIP_IN_WORKTREE)) return null; + const activation = resolveWorktreeRunExecutionActivation( + await instanceSettingsService(db).getExperimental(), + env.PAPERCLIP_INSTANCE_ID?.trim() || null, + ); + const cutoff = activation.armed ? new Date(activation.cutoff) : null; + return cutoff && !Number.isNaN(cutoff.getTime()) ? cutoff : null; +} + +/** + * Board reads repeat while the line changes only when a run starts, finishes or + * joins. A short per-agent memo, shared by concurrent reads, keeps a deep queue + * from being re-read on every task and run read. + */ +export const AGENT_QUEUE_LINE_TTL_MS = 5_000; +const lineCache = new Map }>(); + +function cachedAgentQueuedRunLine( + db: Db, + issuesSvc: Parameters[1], + agent: Agent, + now: Date, +): Promise { + const key = `${agent.companyId}:${agent.id}`; + const at = Date.now(); + const cached = lineCache.get(key); + if (cached && at - cached.at < AGENT_QUEUE_LINE_TTL_MS) return cached.line; + for (const [entryKey, entry] of lineCache) { + if (at - entry.at >= AGENT_QUEUE_LINE_TTL_MS) lineCache.delete(entryKey); + } + const line = agentQueuedRunLine(db, issuesSvc, agent, now); + lineCache.set(key, { at, line }); + line.catch(() => { if (lineCache.get(key)?.line === line) lineCache.delete(key); }); + return line; +} + +/** Test hook: forget memoized lines. */ +export function clearAgentQueueLineCache() { + lineCache.clear(); +} diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index e6f3455bf8f..dfa75c8734c 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -3593,13 +3593,20 @@ async function activeRunMapForIssues( return map; } +/** + * `live` counts descendants with a queued or running run: work that covers a + * blocked parent. `running` counts only descendants whose run started; a run + * still waiting for its agent's slot is not running below. + */ +type LiveDescendantCounts = { live: number; running: number }; + async function liveDescendantCountMapForIssues( dbOrTx: any, companyId: string, issueIds: string[], -): Promise> { +): Promise> { const uniqueIssueIds = [...new Set(issueIds)]; - const map = new Map(); + const map = new Map(); if (uniqueIssueIds.length === 0) return map; for (const issueIdChunk of chunkList( @@ -3615,8 +3622,8 @@ async function liveDescendantCountMapForIssues( target_issues(issue_id) AS ( VALUES ${sql.join(targetRows, sql`, `)} ), - live_issues(live_issue_id, parent_id) AS ( - SELECT DISTINCT live_issue.id, live_issue.parent_id + live_issues(live_issue_id, parent_id, started) AS ( + SELECT DISTINCT live_issue.id, live_issue.parent_id, (live_run.status = 'running' OR live_run.started_at IS NOT NULL) FROM issues live_issue JOIN heartbeat_runs live_run ON live_run.id = live_issue.execution_run_id WHERE live_issue.company_id = ${companyId} @@ -3625,7 +3632,7 @@ async function liveDescendantCountMapForIssues( AND live_run.company_id = ${companyId} AND live_run.status IN ('queued', 'running') UNION - SELECT DISTINCT live_issue.id, live_issue.parent_id + SELECT DISTINCT live_issue.id, live_issue.parent_id, (live_run.status = 'running' OR live_run.started_at IS NOT NULL) FROM heartbeat_runs live_run JOIN issues live_issue ON live_issue.id::text = (live_run.context_snapshot ->> 'issueId') WHERE live_issue.company_id = ${companyId} @@ -3634,8 +3641,8 @@ async function liveDescendantCountMapForIssues( AND live_run.company_id = ${companyId} AND live_run.status IN ('queued', 'running') ), - live_ancestors(live_issue_id, ancestor_id, next_parent_id, visited_issue_ids) AS ( - SELECT live_issues.live_issue_id, parent.id, parent.parent_id, ARRAY[live_issues.live_issue_id, parent.id] + live_ancestors(live_issue_id, started, ancestor_id, next_parent_id, visited_issue_ids) AS ( + SELECT live_issues.live_issue_id, live_issues.started, parent.id, parent.parent_id, ARRAY[live_issues.live_issue_id, parent.id] FROM live_issues JOIN issues parent ON parent.id = live_issues.parent_id WHERE parent.company_id = ${companyId} @@ -3644,6 +3651,7 @@ async function liveDescendantCountMapForIssues( UNION ALL SELECT live_ancestors.live_issue_id, + live_ancestors.started, parent.id, parent.parent_id, live_ancestors.visited_issue_ids || parent.id @@ -3656,7 +3664,8 @@ async function liveDescendantCountMapForIssues( ) SELECT live_ancestors.ancestor_id::text AS "issueId", - count(DISTINCT live_ancestors.live_issue_id)::int AS "liveDescendantCount" + count(DISTINCT live_ancestors.live_issue_id)::int AS "liveDescendantCount", + count(DISTINCT live_ancestors.live_issue_id) FILTER (WHERE live_ancestors.started)::int AS "runningDescendantCount" FROM live_ancestors JOIN target_issues ON target_issues.issue_id = live_ancestors.ancestor_id WHERE live_ancestors.ancestor_id <> live_ancestors.live_issue_id @@ -3669,14 +3678,10 @@ async function liveDescendantCountMapForIssues( for (const row of resultRows) { if (typeof row !== "object" || row === null) continue; const issueId = (row as { issueId?: unknown }).issueId; - const liveDescendantCount = (row as { liveDescendantCount?: unknown }) - .liveDescendantCount; if (typeof issueId !== "string") continue; - const count = - typeof liveDescendantCount === "number" - ? liveDescendantCount - : Number(liveDescendantCount); - if (Number.isFinite(count)) map.set(issueId, count); + const live = Number((row as { liveDescendantCount?: unknown }).liveDescendantCount); + const running = Number((row as { runningDescendantCount?: unknown }).runningDescendantCount); + if (Number.isFinite(live)) map.set(issueId, { live, running: Number.isFinite(running) ? running : 0 }); } } @@ -6864,7 +6869,7 @@ async function listBlockedInboxIssues( listIssueBlockedInboxAttentionMap(dbOrTx, companyId, withRuns, readExternalReviewWaits), includeLiveDescendantSummary ? liveDescendantCountMapForIssues(dbOrTx, companyId, issueIds) - : Promise.resolve(new Map()), + : Promise.resolve(new Map()), ]); const rawSearchInput = filters?.q?.trim() ?? ""; @@ -6937,7 +6942,9 @@ async function listBlockedInboxIssues( ...(includeLiveDescendantSummary ? { liveDescendantCount: - liveDescendantCountByIssueId.get(row.id) ?? 0, + liveDescendantCountByIssueId.get(row.id)?.live ?? 0, + runningDescendantCount: + liveDescendantCountByIssueId.get(row.id)?.running ?? 0, } : {}), ...(contextUserId @@ -8801,7 +8808,7 @@ export function issueService(db: Db) { : Promise.resolve(new Map()), includeLiveDescendantSummary ? timePhase("live_descendants", () => liveDescendantCountMapForIssues(db, companyId, issueIds)) - : Promise.resolve(new Map()), + : Promise.resolve(new Map()), ]); const statsByIssueId = new Map( statsRows.map((row) => [row.issueId, row]), @@ -8857,7 +8864,9 @@ export function issueService(db: Db) { ...(includeLiveDescendantSummary ? { liveDescendantCount: - liveDescendantCountByIssueId.get(row.id) ?? 0, + liveDescendantCountByIssueId.get(row.id)?.live ?? 0, + runningDescendantCount: + liveDescendantCountByIssueId.get(row.id)?.running ?? 0, } : {}), }; @@ -8900,7 +8909,9 @@ export function issueService(db: Db) { ...(includeLiveDescendantSummary ? { liveDescendantCount: - liveDescendantCountByIssueId.get(row.id) ?? 0, + liveDescendantCountByIssueId.get(row.id)?.live ?? 0, + runningDescendantCount: + liveDescendantCountByIssueId.get(row.id)?.running ?? 0, } : {}), ...deriveIssueUserContext(row, contextUserId, { diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 17c1021cc14..d400e260b70 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -268,7 +268,7 @@ "packages/shared/src/home-paths.ts": "fa4f1c440653b76b3dbb7a51026d557287deb45d", "packages/shared/src/humanize-connection.test.ts": "ba9e90a44c63452b0bf9f733adb90241921e8eca", "packages/shared/src/humanize-connection.ts": "fbd414476a0ef7106a7c80d8572208781b091699", - "packages/shared/src/index.ts": "b26e445cb6bcb0d5f0f7d12d5be7840221a4009b", + "packages/shared/src/index.ts": "17f32ce6e701bef6a98cdf685ab1616ab10e99b9", "packages/shared/src/issue-activity-identity.test.ts": "3dc294640f54c2be67943e3e0149cf4eff2c95ca", "packages/shared/src/issue-activity-identity.ts": "1b0f07ec3fe0506aaaf80a4fa9893f3baf8f0075", "packages/shared/src/issue-attribution.test.ts": "84563685e7f3c48029243cf1cc24a3a09ee3cde9", @@ -396,7 +396,7 @@ "packages/shared/src/types/email.ts": "556ff6a0dcefa01eb704d6d00d3b191de81ff4dc", "packages/shared/src/types/environment.ts": "a380b9ce9d2d1763011e9adaa58615a3ef1e56c5", "packages/shared/src/types/execution-continuation.ts": "5afe62afd227efa03ff847d8f73291c14707ab5b", - "packages/shared/src/types/execution-projection.ts": "247fd3cc8e10fef3f2ebad65d6c1148e1d4c68ab", + "packages/shared/src/types/execution-projection.ts": "7a4b8567cb1a8204f9e9f878eb594b745a0428cc", "packages/shared/src/types/external-object.ts": "7a225e0f5a6777d449019a30530c858b753eb4c1", "packages/shared/src/types/feedback.ts": "c5eb31545c7f67c04ad9afd9db45a6b4774bcb35", "packages/shared/src/types/finance.ts": "5c2208c7950020f1005aa0dd86a1680c54dd4354", @@ -410,7 +410,7 @@ "packages/shared/src/types/index.ts": "f63f67250d6d5a0b737478c634244ae3aba0328a", "packages/shared/src/types/instance.ts": "ddabd7cc0d885b5fdf45b33a7937b7b704f8fb7d", "packages/shared/src/types/issue-tree-control.ts": "75489fffd491b8350b05caf57d7796419797d9df", - "packages/shared/src/types/issue.ts": "725bd6f677b9d461460697d1ce10dee739a66ebe", + "packages/shared/src/types/issue.ts": "1a59939658d0b28807b8018bacf83a3ded4fef74", "packages/shared/src/types/live.ts": "b90e55e4de8600c6fc17d7fa8d89079d4ae655ec", "packages/shared/src/types/native-finalization.ts": "c026a73003ae232fea4f7624d561493b383df358", "packages/shared/src/types/plugin.ts": "d009ee12f56e0e168d420cd46bdb8dbea323824c", diff --git a/ui/src/components/AgentQueueWait.tsx b/ui/src/components/AgentQueueWait.tsx new file mode 100644 index 00000000000..e753f8cd6bf --- /dev/null +++ b/ui/src/components/AgentQueueWait.tsx @@ -0,0 +1,67 @@ +import type { ExecutionAgentQueue } from "@paperclipai/shared"; +import { Hourglass } from "lucide-react"; +import { Link } from "@/lib/router"; +import { agentQueueConfigureHref, agentQueueWaitParts } from "@/lib/agent-queue-wait"; +import { cn } from "@/lib/utils"; + +/** + * A run that waits for one of its agent's concurrency slots: + * "Waiting for Codie · 20 slots max · 41 of 295 in line · Configure". + * It is a quiet system wait, not work in progress: no spinner, no elapsed timer. + */ +export function AgentQueueWait({ + queue, + showConfigure = true, + className, +}: { + queue: ExecutionAgentQueue; + showConfigure?: boolean; + className?: string; +}) { + const [waiting, limit, line] = agentQueueWaitParts(queue); + return ( + + + + {waiting} + + {limit} + + {line} + {showConfigure ? ( + <> + + + Configure + + + ) : null} + + ); +} + +/** List-row form: one muted word, the full sentence on hover. */ +export function AgentQueueWaitChip({ queue }: { queue: ExecutionAgentQueue | null }) { + const title = queue ? agentQueueWaitParts(queue).join(" · ") : "Waiting to start"; + return ( + + + ); +} diff --git a/ui/src/components/IssueColumns.tsx b/ui/src/components/IssueColumns.tsx index 54be14b99f0..87b10310ffd 100644 --- a/ui/src/components/IssueColumns.tsx +++ b/ui/src/components/IssueColumns.tsx @@ -1,4 +1,6 @@ import { AgentIdentity } from "@/components/AgentIdentity"; +import { AgentQueueWaitChip } from "./AgentQueueWait"; +import type { IssueWaiting } from "../lib/liveIssueIds"; import type { AvatarAgent } from "./AgentAvatar"; import type { ReactNode } from "react"; import { deriveOriginatingActor, type Issue } from "@paperclipai/shared"; @@ -183,6 +185,7 @@ export function IssueColumnPicker({ export function InboxIssueMetaLeading({ issue, isLive, + waiting = null, subtreeLiveCount = 0, showSubtreeLiveChip = true, showStatus = true, @@ -192,6 +195,8 @@ export function InboxIssueMetaLeading({ }: { issue: Issue; isLive: boolean; + /** Set when every live run of the task is still waiting to start. */ + waiting?: IssueWaiting | null; subtreeLiveCount?: number; showSubtreeLiveChip?: boolean; showStatus?: boolean; @@ -216,7 +221,9 @@ export function InboxIssueMetaLeading({ {issue.identifier ?? issue.id.slice(0, 8)} ) : null} - {isLive && ( + {isLive && waiting ? ( + + ) : isLive && ( {statusLabel(run.status)} - {run.isLive ? ( + {run.isLive && !isWaitingToStart(run) ? ( live @@ -1005,7 +1008,9 @@ export function IssueRunLedgerContent({ /> ) : null} - {nextAction ? ( + {agentQueue ? ( + + ) : nextAction ? (
Next action:{" "} diff --git a/ui/src/components/IssuesList.tsx b/ui/src/components/IssuesList.tsx index 48c6881f6ab..4629dbfe073 100644 --- a/ui/src/components/IssuesList.tsx +++ b/ui/src/components/IssuesList.tsx @@ -59,7 +59,7 @@ import { } from "../lib/inbox"; import { cn, formatDurationMs, formatTokens } from "../lib/utils"; import { SHOW_TASK_PRIORITY_UI } from "../lib/ui-flags"; -import { collectSubtreeLiveCounts } from "../lib/liveIssueIds"; +import { collectSubtreeLiveCounts, issueWaitingOf } from "../lib/liveIssueIds"; import { InboxIssueMetaLeading, InboxIssueTrailingColumns, @@ -1147,7 +1147,8 @@ function StreamlinedIssuesList({ ); const availableIssueColumnSet = useMemo(() => new Set(availableIssueColumns), [availableIssueColumns]); const subtreeLiveCounts = useMemo( - () => collectSubtreeLiveCounts(issues, liveIssueIds ?? new Set()), + // The row chip counts only started sub-tasks; waiting ones are not running below. + () => collectSubtreeLiveCounts(issues, liveIssueIds ?? new Set(), { startedOnly: true }), [issues, liveIssueIds], ); const visibleTrailingIssueColumns = useMemo( @@ -2376,6 +2377,7 @@ function StreamlinedIssuesList({ new Set(availableIssueColumns), [availableIssueColumns]); const subtreeLiveCounts = useMemo( - () => collectSubtreeLiveCounts(issues, liveIssueIds ?? new Set()), + // The row chip counts only started sub-tasks; waiting ones are not running below. + () => collectSubtreeLiveCounts(issues, liveIssueIds ?? new Set(), { startedOnly: true }), [issues, liveIssueIds], ); const visibleTrailingIssueColumns = useMemo( @@ -2229,6 +2230,7 @@ export function IssuesList({ !isWaitingToStart(run)).length; const showWorkspacesLink = experimentalSettings?.enableIsolatedWorkspaces === true; const showApps = experimentalSettings?.enableApps === true; const showStatusCards = experimentalSettings?.enableStatusCards === true; diff --git a/ui/src/components/Sidebar.tsx b/ui/src/components/Sidebar.tsx index 9bc70d1157c..8d4a9df0b3a 100644 --- a/ui/src/components/Sidebar.tsx +++ b/ui/src/components/Sidebar.tsx @@ -1,4 +1,5 @@ import { CoordinatorSidebar } from "./CoordinatorSidebar"; +import { isWaitingToStart } from "../lib/agent-queue-wait"; import { useCoordinatorWorkspaceEnabled } from "@/hooks/useCoordinatorWorkspaceEnabled"; import { Inbox, @@ -94,7 +95,7 @@ function ClassicSidebar({ children }: { children?: ReactNode }) { refetchInterval: sharedLiveRuns.refetchInterval, }); usePublishSharedQueryData(sharedLiveRuns, liveRuns, liveRunsUpdatedAt); - const liveRunCount = liveRuns?.length ?? 0; + const liveRunCount = (liveRuns ?? []).filter((run) => !isWaitingToStart(run)).length; const liveIssueIds = new Set( (liveRuns ?? []).flatMap((run) => run.issueId ? [run.issueId] : []), ); diff --git a/ui/src/components/SidebarAgents.production.tsx b/ui/src/components/SidebarAgents.production.tsx index 08231f68b7d..762bc3846a9 100644 --- a/ui/src/components/SidebarAgents.production.tsx +++ b/ui/src/components/SidebarAgents.production.tsx @@ -1,4 +1,5 @@ import { usePrimaryAgentPresentation } from "@/components/primary-agent/PrimaryAgentPresentation"; +import { isWaitingToStart } from "../lib/agent-queue-wait"; import { AgentAvatar } from "@/components/AgentAvatar"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Link, useLocation } from "@/lib/router"; @@ -359,18 +360,17 @@ export function SidebarAgents({ streamlined = false }: { streamlined?: boolean } const liveCountByAgent = useMemo(() => { const counts = new Map(); + // "N live" counts work that started; queued runs only wait for a slot. for (const run of liveRuns ?? []) { + if (isWaitingToStart(run)) continue; counts.set(run.agentId, (counts.get(run.agentId) ?? 0) + 1); } return counts; }, [liveRuns]); - const liveAgentIds = useMemo(() => { - const ids = new Set(); - for (const [agentId, count] of liveCountByAgent) { - if (count > 0) ids.add(agentId); - } - return ids; - }, [liveCountByAgent]); + const liveAgentIds = useMemo( + () => new Set((liveRuns ?? []).map((run) => run.agentId)), + [liveRuns], + ); const visibleAgents = useMemo(() => { const filtered = (agents ?? []).filter( diff --git a/ui/src/components/SidebarAgents.tsx b/ui/src/components/SidebarAgents.tsx index 7f5f6a55a6d..74d38074424 100644 --- a/ui/src/components/SidebarAgents.tsx +++ b/ui/src/components/SidebarAgents.tsx @@ -1,4 +1,5 @@ import { usePrimaryAgentPresentation } from "@/components/primary-agent/PrimaryAgentPresentation"; +import { isWaitingToStart } from "../lib/agent-queue-wait"; import { AgentAvatar } from "@/components/AgentAvatar"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Link, useLocation } from "@/lib/router"; @@ -359,18 +360,17 @@ export function SidebarAgents({ streamlined = false }: { streamlined?: boolean } const liveCountByAgent = useMemo(() => { const counts = new Map(); + // "N live" counts work that started; queued runs only wait for a slot. for (const run of liveRuns ?? []) { + if (isWaitingToStart(run)) continue; counts.set(run.agentId, (counts.get(run.agentId) ?? 0) + 1); } return counts; }, [liveRuns]); - const liveAgentIds = useMemo(() => { - const ids = new Set(); - for (const [agentId, count] of liveCountByAgent) { - if (count > 0) ids.add(agentId); - } - return ids; - }, [liveCountByAgent]); + const liveAgentIds = useMemo( + () => new Set((liveRuns ?? []).map((run) => run.agentId)), + [liveRuns], + ); const visibleAgents = useMemo(() => { const filtered = (agents ?? []).filter( diff --git a/ui/src/components/task-chat/TaskChatLiveRunPill.test.tsx b/ui/src/components/task-chat/TaskChatLiveRunPill.test.tsx index 363636431e1..70559fab815 100644 --- a/ui/src/components/task-chat/TaskChatLiveRunPill.test.tsx +++ b/ui/src/components/task-chat/TaskChatLiveRunPill.test.tsx @@ -5,6 +5,9 @@ import { createRoot, type Root } from "react-dom/client"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ExecutionProjection } from "@paperclipai/shared"; import type { TranscriptEntry } from "../../adapters"; +import { MemoryRouter } from "@/lib/router"; + +vi.mock("@/context/CompanyContext", () => ({ useCompany: () => ({ selectedCompany: null }) })); import { TaskChatLiveRunPill, toolCountSummaryFromEntries } from "./TaskChatLiveRunPill"; function toolCall(overrides: Partial>): TranscriptEntry { @@ -101,6 +104,29 @@ describe("TaskChatLiveRunPill", () => { }, ); + it("shows a queued run that waits for an agent slot as waiting, not working", () => { + const execution = { + phase: "queued", label: "Waiting for Codie", cause: "agent_at_capacity", + agentQueue: { agentId: "agent-1", agentName: "Codie", maxConcurrentRuns: 20, runningCount: 20, position: 41, queuedCount: 295 }, + } as ExecutionProjection; + act(() => root.render( + + + , + )); + expect(container.textContent).toContain("Waiting for Codie·20 slots max·41 of 295 in line·Configure"); + expect(container.textContent).not.toContain("Working"); + expect(container.querySelector(".animate-spin")).toBeNull(); + expect(container.querySelector("a")?.getAttribute("href")).toBe("/agents/codie/advanced"); + }); + + it("names another server hold on a queued run instead of Working", () => { + const execution = { phase: "queued", label: "Waiting for maintenance", cause: "task_drain" } as ExecutionProjection; + act(() => root.render()); + expect(container.textContent).toContain("Waiting for maintenance"); + expect(container.textContent).not.toContain("Working"); + }); + it("settles to a static 'Worked' summary once the run is terminal", () => { const startedAtMs = 1_000; act(() => { diff --git a/ui/src/components/task-chat/TaskChatLiveRunPill.tsx b/ui/src/components/task-chat/TaskChatLiveRunPill.tsx index ed5ee45955e..e0e4194a44d 100644 --- a/ui/src/components/task-chat/TaskChatLiveRunPill.tsx +++ b/ui/src/components/task-chat/TaskChatLiveRunPill.tsx @@ -7,6 +7,8 @@ import { formatDurationWords } from "@/lib/issue-chat-messages"; import { isCommandTool } from "@/lib/transcriptPresentation"; import { isTerminalRunStatus } from "@/components/task-chat/transcript-adapter"; import { useTaskChatShimmer } from "./TaskChatShimmer"; +import { AgentQueueWait } from "@/components/AgentQueueWait"; +import { agentQueueWaitOf } from "@/lib/agent-queue-wait"; /** * "ran N commands, called M tools" for the live tail's status pill, counted off @@ -46,6 +48,7 @@ export function toolCountSummaryFromEntries(entries: readonly TranscriptEntry[]) */ export function TaskChatLiveRunPill({ status, + execution, startedAtMs, finishedAtMs, toolSummary, @@ -60,9 +63,12 @@ export function TaskChatLiveRunPill({ }) { const active = !isTerminalRunStatus(status); const shimmer = useTaskChatShimmer("working"); + const agentQueue = status === "queued" ? agentQueueWaitOf(execution) : null; + // A specific server hold (maintenance, agent setup) names the wait instead of "Working". + const holdLabel = status === "queued" && execution?.cause ? execution.label : null; // One shared page-wide ticker drives the live elapsed readout, matching the // default view's `useLiveElapsed`. - useSecondTick(active && startedAtMs != null); + useSecondTick(active && startedAtMs != null && !agentQueue); const elapsedMs = startedAtMs == null ? null : (active ? Date.now() : finishedAtMs ?? Date.now()) - startedAtMs; @@ -70,9 +76,17 @@ export function TaskChatLiveRunPill({ ? formatDurationWords(elapsedMs) : null; const failed = ["failed", "timed_out", "cancelled", "interrupted"].includes(status); - const verb = active ? "Working" : failed ? "Stopped" : "Worked"; + const verb = active ? holdLabel ?? "Working" : failed ? "Stopped" : "Worked"; const suffix = elapsed ? `for ${elapsed}` : null; + if (agentQueue) { + return ( +
+ +
+ ); + } + return (
) { + return status === "queued" ? agentQueueWaitOf(execution) : null; +} + /** One identity stays mounted while admission discovers the run's renderer. */ export function TaskChatRunnerHeader({ statusContent, ...props }: RunnerHeaderProps & { statusContent?: ReactNode }) { return
{props.agentName ? : null} - {statusContent ?? : null) ?? }
; } diff --git a/ui/src/lib/agent-queue-wait.ts b/ui/src/lib/agent-queue-wait.ts new file mode 100644 index 00000000000..58793375e23 --- /dev/null +++ b/ui/src/lib/agent-queue-wait.ts @@ -0,0 +1,29 @@ +import type { ExecutionAgentQueue, ExecutionProjection } from "@paperclipai/shared"; +import { agentDetailHref } from "../pages/agent-detail-navigation"; +import { agentRouteRef } from "./utils"; + +/** The server's explanation that a run waits for a free slot of its agent. */ +export function agentQueueWaitOf( + execution: Pick | null | undefined, +): ExecutionAgentQueue | null { + return execution?.cause === "agent_at_capacity" && execution.agentQueue ? execution.agentQueue : null; +} + +/** A run the server accepted but never started: it waits, it is not working. */ +export function isWaitingToStart(run: { status: string; startedAt?: string | Date | null }): boolean { + return run.status === "queued" && !run.startedAt; +} + +/** "Waiting for Codie", "20 slots max", "41 of 295 in line". */ +export function agentQueueWaitParts(queue: ExecutionAgentQueue): [string, string, string] { + return [ + `Waiting for ${queue.agentName}`, + `${queue.maxConcurrentRuns} ${queue.maxConcurrentRuns === 1 ? "slot" : "slots"} max`, + `${queue.position} of ${queue.queuedCount} in line`, + ]; +} + +/** Where the agent's concurrency limit is edited (its run policy, in the Advanced view). */ +export function agentQueueConfigureHref(queue: Pick): string { + return agentDetailHref(agentRouteRef({ id: queue.agentId, name: queue.agentName }), "advanced"); +} diff --git a/ui/src/lib/inbox-live-descendants.test.ts b/ui/src/lib/inbox-live-descendants.test.ts index 55cf6ea7a76..cca5f41b9bf 100644 --- a/ui/src/lib/inbox-live-descendants.test.ts +++ b/ui/src/lib/inbox-live-descendants.test.ts @@ -1,9 +1,6 @@ import { describe, expect, it } from "vitest"; import type { Issue, IssueBlockerAttention } from "@paperclipai/shared"; -import { - resolveInboxIssueBlockerAttention, - resolveIssueLiveDescendantCount, -} from "./inbox-live-descendants"; +import { resolveInboxIssueBlockerAttention, resolveIssueLiveDescendantCount, resolveIssueRunningDescendantCount } from "./inbox-live-descendants"; function makeIssue(overrides: Partial): Issue { return { @@ -88,3 +85,14 @@ describe("inbox live descendant status helpers", () => { ).toBeNull(); }); }); + +describe("resolveIssueRunningDescendantCount", () => { + it("counts only started sub-tasks for the chip and keeps live coverage separate", () => { + const issue = { liveDescendantCount: 3, runningDescendantCount: 1 }; + expect(resolveIssueRunningDescendantCount(issue, 0)).toBe(1); + expect(resolveIssueRunningDescendantCount(issue, 2)).toBe(2); + expect(resolveIssueLiveDescendantCount(issue, 0)).toBe(3); + // Older servers send only the live count. + expect(resolveIssueRunningDescendantCount({ liveDescendantCount: 2 }, 0)).toBe(2); + }); +}); diff --git a/ui/src/lib/inbox-live-descendants.ts b/ui/src/lib/inbox-live-descendants.ts index 36f53a6edee..81ef988e527 100644 --- a/ui/src/lib/inbox-live-descendants.ts +++ b/ui/src/lib/inbox-live-descendants.ts @@ -1,5 +1,16 @@ import type { Issue, IssueBlockerAttention } from "@paperclipai/shared"; +/** The "n live below" chip: descendants whose run started (server count, else live count). */ +export function resolveIssueRunningDescendantCount( + issue: Pick, + loadedSubtreeRunningCount = 0, +): number { + return Math.max( + normalizeLiveDescendantCount(issue.runningDescendantCount ?? issue.liveDescendantCount), + normalizeLiveDescendantCount(loadedSubtreeRunningCount), + ); +} + type InboxLiveDescendantIssue = Pick; interface InboxLiveDescendantOptions { diff --git a/ui/src/lib/liveIssueIds.test.ts b/ui/src/lib/liveIssueIds.test.ts index 06bd8e025c3..e09793726ab 100644 --- a/ui/src/lib/liveIssueIds.test.ts +++ b/ui/src/lib/liveIssueIds.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import type { LiveRunForIssue } from "../api/heartbeats"; -import { collectLiveIssueIds, collectSubtreeLiveCounts } from "./liveIssueIds"; +import { collectLiveIssueIds, collectSubtreeLiveCounts, issueWaitingOf } from "./liveIssueIds"; function liveRun(overrides: Partial): LiveRunForIssue { return { @@ -204,3 +204,30 @@ describe("collectSubtreeLiveCounts", () => { expect(counts.has("a")).toBe(false); }); }); + +describe("issueWaitingOf", () => { + const queue = { agentId: "agent", agentName: "Codie", maxConcurrentRuns: 20, runningCount: 20, position: 41, queuedCount: 295 }; + const waiting = (overrides: Partial) => liveRun({ status: "queued", startedAt: null, ...overrides }); + + it("marks a task whose live runs have not started as waiting, with the agent's line", () => { + const ids = collectLiveIssueIds([ + waiting({ id: "plain", issueId: "a" }), + waiting({ id: "line", issueId: "a", execution: { cause: "agent_at_capacity", agentQueue: queue } as LiveRunForIssue["execution"] }), + ]); + expect(ids.has("a")).toBe(true); + expect(issueWaitingOf(ids, "a")).toEqual({ queue }); + }); + + it("keeps a task with a started run live", () => { + const ids = collectLiveIssueIds([waiting({ id: "queued", issueId: "a" }), liveRun({ id: "running", issueId: "a" })]); + expect(issueWaitingOf(ids, "a")).toBeNull(); + expect(issueWaitingOf(new Set(["a"]), "a")).toBeNull(); + }); + + it("counts a waiting sub-task as live work, but not as running below", () => { + const ids = collectLiveIssueIds([waiting({ id: "queued", issueId: "child" })]); + const tree = [{ id: "parent", parentId: null }, { id: "child", parentId: "parent" }]; + expect(collectSubtreeLiveCounts(tree, ids).get("parent")).toBe(1); + expect(collectSubtreeLiveCounts(tree, ids, { startedOnly: true }).size).toBe(0); + }); +}); diff --git a/ui/src/lib/liveIssueIds.ts b/ui/src/lib/liveIssueIds.ts index a8c83ff9de7..0d106503a07 100644 --- a/ui/src/lib/liveIssueIds.ts +++ b/ui/src/lib/liveIssueIds.ts @@ -1,4 +1,5 @@ -import type { IssueStatus } from "@paperclipai/shared"; +import type { ExecutionAgentQueue, IssueStatus } from "@paperclipai/shared"; +import { agentQueueWaitOf, isWaitingToStart } from "./agent-queue-wait"; import type { LiveRunForIssue } from "../api/heartbeats"; function isLiveRunStatus(status: string): boolean { @@ -58,18 +59,40 @@ function shouldReplaceIssueStatusSnapshot( return !isTerminalIssueStatus(existing.status) && isTerminalIssueStatus(candidate.status); } +/** A task whose live runs have all not started yet; `queue` names the agent's line when the server reports it. */ +export interface IssueWaiting { + queue: ExecutionAgentQueue | null; +} + +const waitingByIssueIds = new WeakMap, ReadonlyMap>(); + export function collectLiveIssueIds( liveRuns: readonly LiveRunForIssue[] | null | undefined, issues?: readonly LiveIssueStatusNode[] | null, ): Set { const ids = new Set(); + const started = new Set(); + const waiting = new Map(); const statusByIssueId = collectIssueStatusById(issues); for (const run of liveRuns ?? []) { - if (run.issueId && isLiveIssueRun(run, statusByIssueId.get(run.issueId))) ids.add(run.issueId); + if (!run.issueId || !isLiveIssueRun(run, statusByIssueId.get(run.issueId))) continue; + ids.add(run.issueId); + if (!isWaitingToStart(run)) started.add(run.issueId); + else if (!waiting.get(run.issueId)?.queue) waiting.set(run.issueId, { queue: agentQueueWaitOf(run.execution) }); } + for (const issueId of started) waiting.delete(issueId); + waitingByIssueIds.set(ids, waiting); return ids; } +/** Whether a live task from {@link collectLiveIssueIds} is only waiting to start. */ +export function issueWaitingOf( + liveIssueIds: ReadonlySet | null | undefined, + issueId: string, +): IssueWaiting | null { + return liveIssueIds ? waitingByIssueIds.get(liveIssueIds)?.get(issueId) ?? null : null; +} + /** * Minimal tree node shape needed to roll live descendants up to their ancestors. * Both list and inbox issue objects satisfy this. @@ -94,6 +117,7 @@ export interface SubtreeLiveNode { export function collectSubtreeLiveCounts( issues: readonly SubtreeLiveNode[] | null | undefined, liveIssueIds: ReadonlySet, + options: { startedOnly?: boolean } = {}, ): Map { const counts = new Map(); if (!issues || issues.length === 0 || liveIssueIds.size === 0) return counts; @@ -102,8 +126,10 @@ export function collectSubtreeLiveCounts( for (const issue of issues) parentById.set(issue.id, issue.parentId); for (const liveId of liveIssueIds) { - // Only roll up live issues that belong to the loaded tree. - if (!parentById.has(liveId)) continue; + // Only roll up live issues that belong to the loaded tree. With + // `startedOnly`, a sub-task that only waits to start is not running below; + // it still counts as live work that covers a blocked parent. + if (!parentById.has(liveId) || (options.startedOnly && issueWaitingOf(liveIssueIds, liveId))) continue; const seen = new Set([liveId]); let parentId = parentById.get(liveId) ?? null; while (parentId && !seen.has(parentId)) { diff --git a/ui/src/pages/Inbox.tsx b/ui/src/pages/Inbox.tsx index 8d5e0c718c3..8fc8013a2e2 100644 --- a/ui/src/pages/Inbox.tsx +++ b/ui/src/pages/Inbox.tsx @@ -44,7 +44,7 @@ import { normalizeIssueFilterState, type IssueFilterState, } from "../lib/issue-filters"; -import { collectLiveIssueIds, collectSubtreeLiveCounts } from "../lib/liveIssueIds"; +import { collectLiveIssueIds, collectSubtreeLiveCounts, issueWaitingOf } from "../lib/liveIssueIds"; import { formatAssigneeUserLabel } from "../lib/assignees"; import { buildCompanyUserLabelMap, buildCompanyUserProfileMap } from "../lib/company-members"; import { @@ -64,7 +64,7 @@ import { } from "../lib/keyboardShortcuts"; import { resolveInboxIssueBlockerAttention, - resolveIssueLiveDescendantCount, + resolveIssueRunningDescendantCount, } from "../lib/inbox-live-descendants"; import { beginLocalInboxArchive, @@ -1822,7 +1822,7 @@ function StreamlinedInbox({ flatNavItemsRef.current = flatNavItems; // Roll live descendant runs up to their ancestors across the loaded inbox tree // so a parent that is not itself live can still surface "n live below". - const subtreeLiveCounts = useMemo(() => { + const subtreeCounts = useMemo(() => { const nodes: { id: string; parentId: string | null }[] = []; const seen = new Set(); const pushIssue = (issue: Issue) => { @@ -1838,8 +1838,12 @@ function StreamlinedInbox({ for (const child of children) pushIssue(child); } } - return collectSubtreeLiveCounts(nodes, liveIssueIds); + return { + live: collectSubtreeLiveCounts(nodes, liveIssueIds), + running: collectSubtreeLiveCounts(nodes, liveIssueIds, { startedOnly: true }), + }; }, [groupedSections, liveIssueIds]); + const subtreeLiveCounts = subtreeCounts.live; const topFlatIndex = useMemo(() => { const map = new Map(); flatNavItems.forEach((entry, index) => { @@ -3057,7 +3061,7 @@ function StreamlinedInbox({ originatingActor?.kind === "user" ? originatingActor.viaAgentId ?? null : null; const isLive = liveIssueIds.has(issue.id); const loadedSubtreeLiveCount = subtreeLiveCounts.get(issue.id) ?? 0; - const liveDescendantCount = resolveIssueLiveDescendantCount(issue, loadedSubtreeLiveCount); + const runningDescendantCount = resolveIssueRunningDescendantCount(issue, subtreeCounts.running.get(issue.id) ?? 0); const blockerAttention = resolveInboxIssueBlockerAttention(issue, { isLive, loadedSubtreeLiveCount, @@ -3108,7 +3112,8 @@ function StreamlinedInbox({ >, +): LiveRunSummary { + if (!runs.length) return EMPTY_LIVE_RUN_SUMMARY; + return { + count: runs.length, + waitingOnly: runs.every(run => isWaitingToStart(run)), + agentQueue: runs.map(run => agentQueueWaitOf(run.execution)).find(Boolean) ?? null, + }; +} + function resolveInterruptibleIssueRun( activeRun: ActiveRunForIssue | null | undefined, liveRuns: readonly LiveRunForIssue[] | undefined, @@ -3622,38 +3639,45 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS recordIssueDetailReadiness({ issue: !!loadedIssue, comments: !commentsLoading, interactions: !interactionsLoading, attachments: !attachmentsLoading, "work-products": !workProductsLoading }); }, [issueId, loadedIssue, commentsLoading, interactionsLoading, attachmentsLoading, workProductsLoading]); - const { data: liveRunCount = 0 } = useQuery( + const { data: liveRunSummary = EMPTY_LIVE_RUN_SUMMARY } = useQuery( { queryKey: queryKeys.issues.liveRuns(issueId!), queryFn: () => heartbeatsApi.liveRunsForIssue(issueId!), enabled: !!issueId, refetchInterval: (query) => liveRunRefetchInterval(hasLiveConnection, query.state, 3000), - select: (runs) => runs.length, + select: summarizeLiveRuns, placeholderData: keepPreviousDataForSameQueryTail( issueId ?? "pending", ), }, ); - const { data: hasActiveRun = false } = useQuery< + const liveRunCount = liveRunSummary.count; + const { data: activeRunSummary = EMPTY_LIVE_RUN_SUMMARY } = useQuery< ActiveRunForIssue | null, Error, - boolean + LiveRunSummary >({ queryKey: queryKeys.issues.activeRun(issueId!), queryFn: () => heartbeatsApi.activeRunForIssue(issueId!), enabled: !!issueId && (!!issue?.executionRunId || issue?.status === "in_progress"), refetchInterval: (query) => liveRunCount > 0 ? false : liveRunRefetchInterval(hasLiveConnection, query.state, 3000), - select: (run) => !!run, + select: (run) => summarizeLiveRuns(run ? [run] : []), placeholderData: keepPreviousDataForSameQueryTail( issueId ?? "pending", ), }); + const hasActiveRun = activeRunSummary.count > 0; const resolvedHasActiveRun = issue ? shouldTrackIssueActiveRun(issue) && hasActiveRun : hasActiveRun; const hasLiveRuns = liveRunCount > 0 || resolvedHasActiveRun; + // Every live run is still waiting to start: say so instead of "Live". + const headerWaiting = hasLiveRuns + && (liveRunCount === 0 || liveRunSummary.waitingOnly) + && (!resolvedHasActiveRun || activeRunSummary.waitingOnly); + const headerAgentQueue = liveRunSummary.agentQueue ?? (resolvedHasActiveRun ? activeRunSummary.agentQueue : null); useEffect(() => { if (!hasLiveRuns && locallyQueuedCommentRunIds.size > 0) { setLocallyQueuedCommentRunIds(new Map()); @@ -7738,7 +7762,11 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS {issue.identifier ?? issue.id.slice(0, 8)} ) : null} - {hasLiveRuns && ( + {headerWaiting ? ( + + + + ) : hasLiveRuns && ( { + const subtreeCounts = useMemo(() => { const nodes: { id: string; parentId: string | null }[] = []; const seen = new Set(); const pushIssue = (issue: Issue) => { @@ -1554,8 +1554,12 @@ export function Inbox() { for (const child of children) pushIssue(child); } } - return collectSubtreeLiveCounts(nodes, liveIssueIds); + return { + live: collectSubtreeLiveCounts(nodes, liveIssueIds), + running: collectSubtreeLiveCounts(nodes, liveIssueIds, { startedOnly: true }), + }; }, [groupedSections, liveIssueIds]); + const subtreeLiveCounts = subtreeCounts.live; const topFlatIndex = useMemo(() => { const map = new Map(); flatNavItems.forEach((entry, index) => { @@ -2686,7 +2690,7 @@ export function Inbox() { originatingActor?.kind === "user" ? originatingActor.viaAgentId ?? null : null; const isLive = liveIssueIds.has(issue.id); const loadedSubtreeLiveCount = subtreeLiveCounts.get(issue.id) ?? 0; - const liveDescendantCount = resolveIssueLiveDescendantCount(issue, loadedSubtreeLiveCount); + const runningDescendantCount = resolveIssueRunningDescendantCount(issue, subtreeCounts.running.get(issue.id) ?? 0); const blockerAttention = resolveInboxIssueBlockerAttention(issue, { isLive, loadedSubtreeLiveCount, @@ -2740,7 +2744,8 @@ export function Inbox() {
], + args: { queue: codieQueue }, +} satisfies Meta; +export default meta; +type Story = StoryObj; + +/** The full sentence: who it waits for, the limit, its place in line, and where to change the limit. */ +export const WaitingInLine: Story = { + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText("Waiting for Codie")).toBeVisible(); + await expect(canvas.getByText("20 slots max")).toBeVisible(); + await expect(canvas.getByText("41 of 295 in line")).toBeVisible(); + await expect(canvas.getByRole("link", { name: /Configure Codie/ })).toHaveAttribute("href", expect.stringMatching(/\/agents\/codie\/advanced$/)); + }, +}; + +/** The task chat's live turn: the waiting line replaces "Working" and its timer. */ +export const InTaskChat: Story = { + render: () => <> + } /> + + , + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getAllByText("Waiting for Codie")).toHaveLength(2); + await expect(canvas.queryByText("Working")).not.toBeInTheDocument(); + }, +}; + +/** Task lists and the inbox: a quiet "Waiting" instead of a pulsing "Live". */ +export const InTaskList: Story = { + render: () =>
+ + {issue.title} +
, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByLabelText("Waiting for Codie · 20 slots max · 41 of 295 in line")).toBeVisible(); + await expect(canvas.queryByText("Live")).not.toBeInTheDocument(); + }, +}; + +export const SingleSlot: Story = { + args: { queue: { ...codieQueue, maxConcurrentRuns: 1, runningCount: 1, position: 2, queuedCount: 3 } }, + play: async ({ canvasElement }) => { + await expect(within(canvasElement).getByText("1 slot max")).toBeVisible(); + }, +}; + +export const Mobile: Story = { globals: { viewport: { value: "mobile", isRotated: false } } };