diff --git a/server/src/__tests__/agent-record-read-projections.test.ts b/server/src/__tests__/agent-record-read-projections.test.ts index 7786b017d3e..c11140d3062 100644 --- a/server/src/__tests__/agent-record-read-projections.test.ts +++ b/server/src/__tests__/agent-record-read-projections.test.ts @@ -94,7 +94,7 @@ describe.skipIf(!support.supported)("agent roster read projections", () => { it("keeps detail reads small while returning the selected agent's complete current configuration", async () => { const f = await fixture(); const captured = await observe(observed => agentRecordQueries(observed).getById(f.workerId)); - expect(captured.calls).toBe(2); + expect(captured.calls).toBe(1); expect(captured.bytes).toBeLessThan(8192); expect(captured.result).toMatchObject({ id: f.workerId, companyId: f.companyId, adapterConfig: f.worker.adapterConfig, runtimeConfig: f.worker.runtimeConfig, metadata: f.worker.metadata, permissions: { canCreateAgents: true }, spentMonthlyCents: 19.5, diff --git a/server/src/__tests__/agent-reporting-chain.test.ts b/server/src/__tests__/agent-reporting-chain.test.ts new file mode 100644 index 00000000000..06c13d5bea0 --- /dev/null +++ b/server/src/__tests__/agent-reporting-chain.test.ts @@ -0,0 +1,103 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { expect, it } from "vitest"; +import { agents, companies } from "@paperclipai/db"; +import { getAgentWorkEligibility } from "@paperclipai/shared"; +import { readAgentReportingChain } from "../lib/agent-reporting-chain.js"; +import { evaluateAgentInvokability, evaluateAgentInvokabilityFromDb, type AgentOrgRow } from "../services/agent-invokability.js"; +import { describeEmbeddedPostgres, useEmbeddedPostgres } from "./helpers/route-test-harness.js"; + +const STATUSES = ["active", "idle", "running", "error", "paused", "terminated", "pending_approval", "retired"]; + +function random(seed: number) { + let state = seed >>> 0; + return () => { + state = (state * 1664525 + 1013904223) >>> 0; + return state / 2 ** 32; + }; +} + +describeEmbeddedPostgres("agent reporting chain", () => { + const ctx = useEmbeddedPostgres("agent-reporting-chain-"); + + async function company(name: string) { + const id = randomUUID(); + await ctx.db.insert(companies).values({ id, name, issuePrefix: `C${id.slice(0, 7).toUpperCase()}` }); + return id; + } + + async function generateOrgChart(seed: number) { + const next = random(seed); + const pick = (items: readonly T[]) => items[Math.floor(next() * items.length)]!; + const home = await company(`Home ${seed}`), other = await company(`Other ${seed}`); + const shape = (["deep", "tree", "cycle", "mixed"] as const)[seed % 4]!; + const size = shape === "deep" ? 45 : 2 + Math.floor(next() * 45); + const homeRows: AgentOrgRow[] = Array.from({ length: size }, (_, index) => ({ id: randomUUID(), companyId: home, + name: `agent-${seed}-${index}`, reportsTo: null, status: next() < 0.6 ? "idle" : pick(STATUSES) })); + const otherRows: AgentOrgRow[] = Array.from({ length: 4 }, (_, index) => ({ id: randomUUID(), companyId: other, + name: `other-${seed}-${index}`, reportsTo: null, status: pick(STATUSES) })); + for (const [index, row] of homeRows.entries()) { + const roll = next(); + if (shape === "deep") row.reportsTo = index === 0 ? null : homeRows[index - 1]!.id; + else if (shape === "cycle") row.reportsTo = homeRows[(index + 1) % homeRows.length]!.id; + else if (roll < 0.12) row.reportsTo = null; + else if (roll < 0.2) row.reportsTo = pick(otherRows).id; + else if (roll < 0.27) row.reportsTo = randomUUID(); + else if (roll < 0.35 || shape === "mixed") row.reportsTo = pick(homeRows).id; + else row.reportsTo = index === 0 ? null : homeRows[Math.floor(next() * index)]!.id; + } + for (const row of otherRows) row.reportsTo = next() < 0.5 ? pick(homeRows).id : null; + await ctx.db.transaction(async tx => { + await tx.execute(sql`set local session_replication_role = replica`); + await tx.insert(agents).values([...homeRows, ...otherRows].map(row => ({ ...row, role: "engineer", adapterType: "process", + adapterConfig: {} }))); + }); + return { home, other, shape, rows: [...homeRows, ...otherRows] }; + } + + it("gives the same invokability and org-chain health as the whole-company list for generated org charts", async () => { + const seen = { shapes: new Set(), reasons: new Set(), healths: new Set(), maxDepth: 0 }; + for (let seed = 1; seed <= 60; seed++) { + const chart = await generateOrgChart(seed); + seen.shapes.add(chart.shape); + for (const companyId of [chart.home, chart.other]) { + const companyRows = await ctx.db.select({ id: agents.id, companyId: agents.companyId, name: agents.name, + reportsTo: agents.reportsTo, status: agents.status }).from(agents).where(eq(agents.companyId, companyId)); + for (const agent of companyRows) { + const listBased = evaluateAgentInvokability(agent, companyRows); + expect(await evaluateAgentInvokabilityFromDb(ctx.db, agent), `${chart.shape} ${seed} ${agent.name}`).toEqual(listBased); + const chain = await readAgentReportingChain(ctx.db, agent); + const listHealth = getAgentWorkEligibility({ agent, agents: companyRows }); + expect(getAgentWorkEligibility({ agent, agents: chain }), `${chart.shape} ${seed} ${agent.name}`).toEqual(listHealth); + seen.reasons.add(listBased.invokable ? "invokable" : listBased.reason); + seen.healths.add(listHealth.orgChainHealth.reason); + seen.maxDepth = Math.max(seen.maxDepth, listHealth.orgChainHealth.fullChain.length - 1); + } + } + } + expect([...seen.shapes].sort()).toEqual(["cycle", "deep", "mixed", "tree"]); + expect([...seen.healths].sort()).toEqual(["cycle", "healthy", "missing_manager", "terminated_ancestor"]); + for (const reason of ["invokable", "paused", "terminated", "pending_approval", "unknown_status", "manager_missing", + "manager_terminated", "reporting_cycle"]) expect(seen.reasons).toContain(reason); + expect(seen.maxDepth).toBeGreaterThanOrEqual(40); + }, 300_000); + + it("reads only the agent's own managers, and nothing for a root agent", async () => { + const home = await company("Chain only"); + const ids = Array.from({ length: 50 }, () => randomUUID()); + await ctx.db.insert(agents).values(ids.map((id, index) => ({ id, companyId: home, name: `a${index}`, role: "engineer", + adapterType: "process", adapterConfig: {}, status: "idle", reportsTo: index === 0 ? null : ids[index < 10 ? index - 1 : 0] }))); + const [deep] = await ctx.db.select().from(agents).where(eq(agents.id, ids[9]!)); + expect((await readAgentReportingChain(ctx.db, deep!)).map(row => row.id)).toEqual(ids.slice(0, 9).reverse()); + const [root] = await ctx.db.select().from(agents).where(eq(agents.id, ids[0]!)); + const execute = ctx.db.execute.bind(ctx.db); + let statements = 0; + const counted = Object.assign(Object.create(ctx.db), { execute: (...args: Parameters) => { + statements += 1; return execute(...args); + } }); + expect(await evaluateAgentInvokabilityFromDb(counted, root!)).toEqual({ invokable: true }); + expect(await evaluateAgentInvokabilityFromDb(counted, { ...deep!, status: "paused" })).toMatchObject({ reason: "paused" }); + expect(statements).toBe(0); + expect(evaluateAgentInvokability(deep!, [])).toMatchObject({ reason: "manager_missing" }); + }); +}); diff --git a/server/src/__tests__/helpers/recovery-population.ts b/server/src/__tests__/helpers/recovery-population.ts new file mode 100644 index 00000000000..16947d39334 --- /dev/null +++ b/server/src/__tests__/helpers/recovery-population.ts @@ -0,0 +1,54 @@ +import { agents, heartbeatRuns, issueRecoveryActions, issues, type Db } from "@paperclipai/db"; + +const HOUR = 60 * 60_000; + +export type RecoveryPopulation = { + agentIds: string[]; + pausedAgentId: string; + terminatedAgentId: string; + inReview: string[]; + inProgress: string[]; + todo: string[]; + withRecoveryAction: string[]; + terminated: string[]; +}; + +export async function seedRecoveryPopulation(db: Db, companyId: string, counts: { + agents: number; inReview: number; inProgress?: number; todo?: number; recoveryActions?: number; terminated?: number; +}): Promise { + const config = { adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } }, role: "engineer" }; + const [ceo] = await db.insert(agents).values({ ...config, companyId, name: "Chief", status: "idle" }).returning({ id: agents.id }); + const managers = await db.insert(agents).values(Array.from({ length: 6 }, (_, index) => ({ ...config, companyId, + name: `Manager ${index}`, status: "idle", reportsTo: ceo!.id }))).returning({ id: agents.id }); + const workers = await db.insert(agents).values(Array.from({ length: Math.max(1, counts.agents - 9) }, (_, index) => ({ ...config, + companyId, name: `Worker ${index}`, status: "idle", reportsTo: managers[index % managers.length]!.id }))).returning({ id: agents.id }); + const [paused] = await db.insert(agents).values({ ...config, companyId, name: "Paused worker", status: "paused", + lifecycleHolds: ["manual"], reportsTo: managers[0]!.id }).returning({ id: agents.id }); + const [terminatedAgent] = await db.insert(agents).values({ ...config, companyId, name: "Former worker", status: "terminated", + reportsTo: managers[1]!.id }).returning({ id: agents.id }); + const workerIds = workers.map(row => row.id); + const finishedAt = new Date(Date.now() - HOUR); + + const insertTasks = async (count: number, values: (index: number) => Partial) => count === 0 ? [] + : (await db.insert(issues).values(Array.from({ length: count }, (_, index) => ({ companyId, title: `Task ${index}`, + ...values(index) }))).returning({ id: issues.id, assigneeAgentId: issues.assigneeAgentId })).map(row => row); + + const reviews = await insertTasks(counts.inReview, index => ({ status: "in_review", + assigneeAgentId: workerIds[index % workerIds.length] })); + if (reviews.length) await db.insert(heartbeatRuns).values(reviews.map(task => ({ companyId, agentId: task.assigneeAgentId!, + status: "succeeded", runtimeMode: "native", startedAt: finishedAt, finishedAt, contextSnapshot: { issueId: task.id } }))); + const progressing = await insertTasks(counts.inProgress ?? 0, () => ({ status: "in_progress", assigneeAgentId: paused!.id })); + if (progressing.length) await db.insert(heartbeatRuns).values(progressing.map(task => ({ companyId, agentId: paused!.id, + status: "cancelled", errorCode: "agent_paused", startedAt: finishedAt, finishedAt, contextSnapshot: { issueId: task.id } }))); + const todo = await insertTasks(counts.todo ?? 0, () => ({ status: "todo", assigneeAgentId: paused!.id })); + const terminated = await insertTasks(counts.terminated ?? 0, () => ({ status: "todo", assigneeAgentId: terminatedAgent!.id })); + const withAction = reviews.slice(0, counts.recoveryActions ?? 0); + if (withAction.length) await db.insert(issueRecoveryActions).values(withAction.map((task, index) => ({ companyId, + sourceIssueId: task.id, kind: "missing_disposition", status: "active", ownerType: "agent", ownerAgentId: task.assigneeAgentId, + cause: "seeded", fingerprint: `seeded-${index}`, nextAction: "seeded" }))); + return { agentIds: [ceo!.id, ...managers.map(row => row.id), ...workerIds, paused!.id, terminatedAgent!.id], + pausedAgentId: paused!.id, terminatedAgentId: terminatedAgent!.id, + inReview: reviews.map(row => row.id), inProgress: progressing.map(row => row.id), todo: todo.map(row => row.id), + withRecoveryAction: withAction.map(row => row.id), terminated: terminated.map(row => row.id) }; +} diff --git a/server/src/__tests__/issue-recovery-pass-reads.test.ts b/server/src/__tests__/issue-recovery-pass-reads.test.ts new file mode 100644 index 00000000000..e0b26bcad78 --- /dev/null +++ b/server/src/__tests__/issue-recovery-pass-reads.test.ts @@ -0,0 +1,74 @@ +import { setImmediate as immediate } from "node:timers/promises"; +import { asc, eq } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { issues } from "@paperclipai/db"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; +import { seedRecoveryPopulation } from "./helpers/recovery-population.js"; + +const PASS = 25; + +describeIssueRecovery("stranded-task check reads", () => { + const state = issueRecoveryFixture({ meter: true }); + + async function measure(work: () => Promise) { + const before = state.meter.read(); + state.meter.startLog(); + await work(); + const log = state.meter.stopLog(); + const after = state.meter.read(); + return { statements: after.statements - before.statements, bytes: after.bytesReceived - before.bytesReceived, + rows: after.rows - before.rows, log }; + } + + async function seed() { + const f = await state.fixture(); + const population = await seedRecoveryPopulation(state.db, f.companyId, { agents: 50, inReview: 200 }); + return { ...f, population }; + } + + it("reads a lean agent row and the agent's own reporting chain for a 50-agent, 200-task company", async () => { + const { companyId } = await seed(); + const ordered = await state.db.select({ id: issues.id }).from(issues) + .where(eq(issues.companyId, companyId)).orderBy(asc(issues.id)); + const tasks = ordered.map(row => row.id); + const check = (issueId: string) => state.heartbeat.reconcileStrandedAssignedIssues({ companyId, issueId, + skipExternalConversationSettlement: true, skipRelatedRecovery: true }); + + await check(tasks[0]!); + const single = await measure(() => check(tasks[1]!)); + const passReads = await measure(async () => { for (const issueId of tasks.slice(0, PASS)) await check(issueId); }); + const companyAgentLists = passReads.log.filter(entry => /from "agents" where "agents"\."company_id" = \$1$/.test(entry.sql.trim())); + const fullAgentRows = passReads.log.filter(entry => /^select "id", "company_id", "name", "role"/.test(entry.sql.trim())); + const writes = passReads.log.filter(entry => /^(insert|update|delete)/i.test(entry.sql.trim())); + console.info(JSON.stringify({ singleCheck: { statements: single.statements, bytes: single.bytes, rows: single.rows }, + pass: { checks: PASS, statements: passReads.statements, bytes: passReads.bytes, rows: passReads.rows, + perCheckStatements: passReads.statements / PASS, perCheckBytes: Math.round(passReads.bytes / PASS) }, + companyAgentLists: companyAgentLists.length, fullAgentRows: fullAgentRows.length })); + + expect(writes).toEqual([]); + expect(companyAgentLists).toEqual([]); + expect(fullAgentRows).toEqual([]); + expect(passReads.bytes / PASS).toBeLessThan(10_000); + }, 300_000); + + it("drains a full company load through the real worker with fewer statements and bytes per check", async () => { + const { population } = await seed(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const drained = await measure(async () => { + await state.start({}); + const until = Date.now() + 60_000; + while (Date.now() < until) { + await vi.advanceTimersByTimeAsync(500); + await Promise.allSettled([...state.passes]); await immediate(); + } + }); + const checks = drained.log.filter(entry => /from "case_handoffs"/.test(entry.sql) && /"issues"\."id" = \$/.test(entry.sql)).length; + const companyAgentLists = drained.log.filter(entry => /from "agents" where "agents"\."company_id" = \$1$/.test(entry.sql.trim())); + console.info(JSON.stringify({ worker: { strandedChecks: checks, statements: drained.statements, bytes: drained.bytes, + rows: drained.rows, perCheckStatements: drained.statements / Math.max(1, checks), + perCheckBytes: Math.round(drained.bytes / Math.max(1, checks)), companyAgentLists: companyAgentLists.length } })); + expect(checks).toBeGreaterThanOrEqual(population.inReview.length); + expect(companyAgentLists).toEqual([]); + expect(drained.bytes / checks).toBeLessThan(10_000); + }, 600_000); +}); diff --git a/server/src/__tests__/issue-recovery-review-wait-pass.test.ts b/server/src/__tests__/issue-recovery-review-wait-pass.test.ts new file mode 100644 index 00000000000..6a7e8fe74f0 --- /dev/null +++ b/server/src/__tests__/issue-recovery-review-wait-pass.test.ts @@ -0,0 +1,116 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { expect, it } from "vitest"; +import { chatConversations, chatDeliveries, chatEndpoints, chatMessageLinks, heartbeatRuns, issueComments, issues, + statusDecisions, type Db } from "@paperclipai/db"; +import { listExternalReviewWaits } from "../services/native-runtime/external-review-wait.js"; +import { createRecoveryPassReads } from "../services/recovery/pass-reads.js"; +import { seedBoundChatAuthorization } from "./helpers/bound-chat-authorization.js"; +import { seedGitHubReviewWait } from "./helpers/github-review-wait.js"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; + +const boundRunLinkRead = /from "chat_message_links" inner join "chat_deliveries"/i; + +async function waitOnReply(db: Db, input: { companyId: string; issueId: string; runId: string }) { + const decisionId = randomUUID(); + await db.transaction(async (tx) => { + await tx.execute(sql`set local session_replication_role = replica`); + await tx.insert(statusDecisions).values({ id: decisionId, companyId: input.companyId, issueId: input.issueId, runId: input.runId, + assessmentId: randomUUID(), decisionVersion: 1, policyVersion: "test", fromStatus: "in_progress", toStatus: "in_review", + reasonCode: "external_chat_response_waiting", decisionJson: {}, decisionDigest: decisionId, applicationState: "applied", + appliedAt: new Date() }); + }); + await db.update(issues).set({ status: "in_review", lastStatusDecisionId: decisionId }).where(eq(issues.id, input.issueId)); +} + +describeIssueRecovery("stranded-task external review waits within one pass", () => { + const state = issueRecoveryFixture({ meter: true }); + + async function seed() { + const db = state.db; + const f = await seedBoundChatAuthorization(db); + const otherConversationId = randomUUID(), otherCommentId = randomUUID(), deliveryId = randomUUID(), otherRunId = randomUUID(); + await db.insert(chatConversations).values({ id: otherConversationId, companyId: f.companyId, endpointId: f.endpointId, + resourceId: f.resourceId, issueId: f.otherIssueId, externalConversationId: "channel-1", externalThreadId: "thread-2", + externalLabel: "Other thread", state: "active" }); + await db.insert(issueComments).values({ id: otherCommentId, companyId: f.companyId, issueId: f.otherIssueId, + authorType: "user", authorUserId: f.userId, body: "Another request" }); + await db.insert(chatDeliveries).values({ id: deliveryId, companyId: f.companyId, endpointId: f.endpointId, + conversationId: otherConversationId, principalId: f.principalId, providerEventId: deliveryId, deduplicationKey: deliveryId, + eventKind: "message", normalizedEvent: {}, state: "processed", processedAt: new Date() }); + await db.insert(chatMessageLinks).values({ companyId: f.companyId, endpointId: f.endpointId, conversationId: otherConversationId, + deliveryId, commentId: otherCommentId, providerMessageId: otherCommentId, direction: "inbound" }); + await db.insert(heartbeatRuns).values([ + { id: f.runId, companyId: f.companyId, agentId: f.agentId, status: "succeeded", runtimeMode: "native", nativeIssueId: f.issueId, + finishedAt: new Date(), contextSnapshot: { ...f.context, issueId: f.issueId } }, + { id: otherRunId, companyId: f.companyId, agentId: f.agentId, status: "succeeded", runtimeMode: "native", + nativeIssueId: f.otherIssueId, finishedAt: new Date(), + contextSnapshot: { ...f.context, issueId: f.otherIssueId, wakeCommentIds: [otherCommentId] } }, + ]); + await waitOnReply(db, { companyId: f.companyId, issueId: f.issueId, runId: f.runId }); + await waitOnReply(db, { companyId: f.companyId, issueId: f.otherIssueId, runId: otherRunId }); + + const inherited = await seedGitHubReviewWait(db, { companyId: f.companyId, agentId: f.agentId }); + await db.update(heartbeatRuns).set({ status: "succeeded", runtimeMode: "native", finishedAt: new Date(), + contextSnapshot: { issueId: inherited.issueId } }).where(eq(heartbeatRuns.id, inherited.runId)); + await db.update(chatEndpoints).set({ status: "active" }).where(eq(chatEndpoints.id, inherited.endpointId)); + await waitOnReply(db, { companyId: f.companyId, issueId: inherited.issueId, runId: inherited.runId }); + + const [none] = await db.insert(issues).values({ companyId: f.companyId, title: "No wait", status: "in_review", + assigneeAgentId: f.agentId }).returning({ id: issues.id }); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "succeeded", runtimeMode: "native", + nativeIssueId: none!.id, finishedAt: new Date(), contextSnapshot: { issueId: none!.id } }); + + const foreign = await seedGitHubReviewWait(db); + await db.update(heartbeatRuns).set({ status: "succeeded", runtimeMode: "native", finishedAt: new Date(), + contextSnapshot: { issueId: foreign.issueId } }).where(eq(heartbeatRuns.id, foreign.runId)); + await db.update(chatEndpoints).set({ status: "active" }).where(eq(chatEndpoints.id, foreign.endpointId)); + await waitOnReply(db, { companyId: foreign.companyId, issueId: foreign.issueId, runId: foreign.runId }); + + return { companyId: f.companyId, direct: [f.issueId, f.otherIssueId], inherited: inherited.issueId, none: none!.id, + foreign: { companyId: foreign.companyId, issueId: foreign.issueId } }; + } + + it("answers each task's review wait the same from one read for the pass as from one read per task", async () => { + const f = await seed(); + const ids = [...f.direct, f.inherited, f.none, f.foreign.issueId]; + const single = new Set(); + for (const issueId of ids) { + if ((await listExternalReviewWaits(state.db, f.companyId, [issueId])).length) single.add(issueId); + } + const batched = new Set((await listExternalReviewWaits(state.db, f.companyId, ids)).map(wait => wait.issueId)); + expect(batched).toEqual(single); + for (const issueId of f.direct) expect(single).toContain(issueId); + expect(single).not.toContain(f.none); + expect(single).not.toContain(f.foreign.issueId); + const foreignSingle = (await listExternalReviewWaits(state.db, f.foreign.companyId, [f.foreign.issueId])).length > 0; + const foreignBatched = (await listExternalReviewWaits(state.db, f.foreign.companyId, ids)).some(wait => wait.issueId === f.foreign.issueId); + expect(foreignBatched).toBe(foreignSingle); + }); + + it("reads the review waits of a pass once and skips the same tasks as per-task reads", async () => { + const f = await seed(); + const ids = [...f.direct, f.inherited, f.none]; + const check = (issueId: string, pass?: ReturnType) => + state.heartbeat.reconcileStrandedAssignedIssues({ companyId: f.companyId, issueId, skipExternalConversationSettlement: true, + skipRelatedRecovery: true, ...(pass ? { pass } : {}) }); + const run = async (pass?: ReturnType) => { + state.meter.startLog(); + const results = []; + for (const issueId of ids) { + const { issueIds: _ids, ...counts } = await check(issueId, pass); + results.push(counts); + } + const log = state.meter.stopLog(); + return { results, linkReads: log.filter(entry => boundRunLinkRead.test(entry.sql)).length, statements: log.length }; + }; + const perTask = await run(); + const perPass = await run(createRecoveryPassReads(ids)); + console.info(JSON.stringify({ perTask: { linkReads: perTask.linkReads, statements: perTask.statements }, + perPass: { linkReads: perPass.linkReads, statements: perPass.statements } })); + expect(perPass.results).toEqual(perTask.results); + expect(perTask.linkReads).toBeGreaterThanOrEqual(f.direct.length); + expect(perPass.linkReads).toBeLessThan(perTask.linkReads); + expect(perPass.statements).toBeLessThan(perTask.statements); + }); +}); diff --git a/server/src/__tests__/issue-recovery.test-fixture.ts b/server/src/__tests__/issue-recovery.test-fixture.ts index da4485f62e4..4aa0b6ac764 100644 --- a/server/src/__tests__/issue-recovery.test-fixture.ts +++ b/server/src/__tests__/issue-recovery.test-fixture.ts @@ -11,6 +11,7 @@ import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js import { instanceSettingsService } from "../services/instance-settings.js"; import { issueService } from "../services/issues.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { meterDatabaseStatements } from "./helpers/database-statement-meter.js"; vi.setConfig({ testTimeout: 90_000 }); @@ -18,7 +19,8 @@ const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_RECOVERY_TEST_D const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); export const describeIssueRecovery: ReturnType = describe.skipIf(!support.supported); -export function issueRecoveryFixture() { +export function issueRecoveryFixture(options: { meter?: boolean } = {}) { + let meter: ReturnType | undefined; let database: Awaited> | undefined; let db: ReturnType; let heartbeat: ReturnType; @@ -35,6 +37,7 @@ export function issueRecoveryFixture() { beforeAll(async () => { if (!external) database = await startEmbeddedPostgresTestDatabase("issue-recovery-work-"); db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + if (options.meter) meter = meterDatabaseStatements(db, external ?? database!.connectionString); await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); }, 90_000); beforeEach(async () => { @@ -49,7 +52,7 @@ export function issueRecoveryFixture() { vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]); }); - afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + afterAll(async () => { meter?.uninstall(); await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); async function start(options: { backstopMs?: number } = { backstopMs: 0 }) { coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); @@ -232,6 +235,7 @@ export function issueRecoveryFixture() { return { start, compete, fixture, completingProvider, settled, quiet, finish, completed, savedConversation, get db() { return db; }, + get meter() { return meter!; }, get heartbeat() { return heartbeat; }, set heartbeat(value: typeof heartbeat) { heartbeat = value; }, get coordinator() { return coordinator; }, diff --git a/server/src/__tests__/monthly-spend-service.test.ts b/server/src/__tests__/monthly-spend-service.test.ts index 92a67421fc0..c1531a2d681 100644 --- a/server/src/__tests__/monthly-spend-service.test.ts +++ b/server/src/__tests__/monthly-spend-service.test.ts @@ -54,27 +54,6 @@ describe("monthly spend hydration", () => { it("recomputes agent spentMonthlyCents from the current utc month instead of returning stale stored values", async () => { const dbStub = createSelectSequenceDb([ - [{ - id: "agent-1", - companyId: "company-1", - name: "Budget Agent", - role: "general", - title: null, - reportsTo: null, - capabilities: null, - adapterType: "claude-local", - adapterConfig: {}, - runtimeConfig: {}, - budgetMonthlyCents: 5000, - spentMonthlyCents: 999999, - metadata: null, - permissions: null, - status: "idle", - pauseReason: null, - pausedAt: null, - createdAt: new Date(), - updatedAt: new Date(), - }], [{ id: "agent-1", companyId: "company-1", diff --git a/server/src/lib/agent-records.ts b/server/src/lib/agent-records.ts index b5e2b01b5d4..f5b7500cabf 100644 --- a/server/src/lib/agent-records.ts +++ b/server/src/lib/agent-records.ts @@ -9,6 +9,7 @@ import { import { conflict, notFound, unprocessable } from "../errors.js"; import { REDACTED_EVENT_VALUE, sanitizeRecord } from "../redaction.js"; import { normalizeAgentPermissions } from "./agent-permissions.js"; +import { readAgentReportingChain } from "./agent-reporting-chain.js"; import { builtInAgentMarkersEqual, readBuiltInAgentMarker } from "./built-in-agent-metadata.js"; export function hashToken(token: string) { @@ -417,11 +418,11 @@ export function agentRecordQueries(db: Db) { .where(eq(agents.id, id)) .then((rows) => rows[0] ?? null); if (!row) return null; - const [companyRows, hydrated] = await Promise.all([ - listCompanyAgentEligibilityRows(row.companyId), + const [chainRows, hydrated] = await Promise.all([ + readAgentReportingChain(db, row), hydrateAgentSpend([row]).then((rows) => rows[0]!), ]); - return normalizeAgentRow(hydrated, companyRows); + return normalizeAgentRow(hydrated, chainRows); } async function ensureManager(companyId: string, managerId: string) { diff --git a/server/src/lib/agent-reporting-chain.ts b/server/src/lib/agent-reporting-chain.ts new file mode 100644 index 00000000000..c2ae6340ea5 --- /dev/null +++ b/server/src/lib/agent-reporting-chain.ts @@ -0,0 +1,22 @@ +import { sql } from "drizzle-orm"; +import type { agents, Db } from "@paperclipai/db"; + +export type AgentReportingChainRow = Pick; + +export async function readAgentReportingChain( + db: Pick, + agent: { companyId: string; reportsTo: string | null }, +): Promise { + if (!agent.reportsTo) return []; + const rows = await db.execute<{ id: string; company_id: string; name: string; reports_to: string | null; status: string }>(sql` + with recursive chain(id, company_id, name, reports_to, status) as ( + select id, company_id, name, reports_to, status from agents + where company_id = ${agent.companyId}::uuid and id = ${agent.reportsTo}::uuid + union + select manager.id, manager.company_id, manager.name, manager.reports_to, manager.status + from agents manager inner join chain on manager.id = chain.reports_to + where manager.company_id = ${agent.companyId}::uuid + ) select id::text, company_id::text, name, reports_to::text, status from chain`); + return rows.map(row => ({ id: row.id, companyId: row.company_id, name: row.name, reportsTo: row.reports_to, + status: row.status as AgentReportingChainRow["status"] })); +} diff --git a/server/src/services/agent-invokability.ts b/server/src/services/agent-invokability.ts index b9b39d59f5d..4d455580eab 100644 --- a/server/src/services/agent-invokability.ts +++ b/server/src/services/agent-invokability.ts @@ -1,7 +1,7 @@ import type { Db } from "@paperclipai/db"; import { agents } from "@paperclipai/db"; -import { getAgentWorkEligibility, type AgentEligibilityAgent, type AgentOrgChainHealth } from "@paperclipai/shared"; -import { eq } from "drizzle-orm"; +import { getAgentWorkEligibility, isAgentStatusInvokable, type AgentEligibilityAgent, type AgentOrgChainHealth } from "@paperclipai/shared"; +import { readAgentReportingChain } from "../lib/agent-reporting-chain.js"; type AgentStatus = (typeof agents.$inferSelect)["status"]; @@ -119,18 +119,8 @@ export async function evaluateAgentInvokabilityFromDb( db: Db, agent: AgentOrgRow | null | undefined, ): Promise { - if (!agent) return evaluateAgentInvokability(agent, []); - const companyAgents = await db - .select({ - id: agents.id, - companyId: agents.companyId, - name: agents.name, - reportsTo: agents.reportsTo, - status: agents.status, - }) - .from(agents) - .where(eq(agents.companyId, agent.companyId)); - return evaluateAgentInvokability(agent, companyAgents); + if (!agent || !isAgentStatusInvokable(agent.status)) return evaluateAgentInvokability(agent, []); + return evaluateAgentInvokability(agent, await readAgentReportingChain(db, agent)); } export function listInvalidOrgChainDescendantIds( diff --git a/server/src/services/recovery/pass-reads.ts b/server/src/services/recovery/pass-reads.ts new file mode 100644 index 00000000000..e01c8f13e75 --- /dev/null +++ b/server/src/services/recovery/pass-reads.ts @@ -0,0 +1,5 @@ +export type RecoveryPassReads = { issueIds: ReadonlySet; reads: Map> }; + +export function createRecoveryPassReads(issueIds: Iterable = []): RecoveryPassReads { + return { issueIds: new Set(issueIds), reads: new Map() }; +} diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 7a855a721d0..52b165c4bb5 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -159,7 +159,8 @@ import { buildIssueBlockersResolvedWakeStateKey, findExistingIssueBlockersResolvedWakeForReadyState, } from "../issue-dependency-wakeups.js"; -import { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js"; +import { evaluateAgentInvokabilityFromDb, type AgentOrgRow } from "../agent-invokability.js"; +import { createRecoveryPassReads, type RecoveryPassReads } from "./pass-reads.js"; import { isHeartbeatWakeOnDemandEnabled } from "../heartbeat-policy.js"; import { DEFAULT_MAX_SUCCESSFUL_RUN_HANDOFF_ATTEMPTS, @@ -208,6 +209,17 @@ import { type DbTransaction = Parameters[0]>[0]; type DbOrTransaction = Db | DbTransaction; +const strandedAgentFields = { + id: agents.id, + companyId: agents.companyId, + name: agents.name, + reportsTo: agents.reportsTo, + status: agents.status, + lifecycleState: agents.lifecycleState, + lifecycleHolds: agents.lifecycleHolds, +}; +type StrandedAgent = { [K in keyof typeof strandedAgentFields]: (typeof agents.$inferSelect)[K] }; + const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = [ "queued", "running", @@ -1039,11 +1051,33 @@ export function recoveryService( } async function isAgentInvokable( - agent: typeof agents.$inferSelect | null | undefined, + agent: AgentOrgRow | null | undefined, ) { return (await evaluateAgentInvokabilityFromDb(db, agent)).invokable; } + async function readStrandedAgent(agentId: string, companyId: string) { + const agent: StrandedAgent | null = await db.select(strandedAgentFields).from(agents).where(eq(agents.id, agentId)) + .then((rows) => rows[0] ?? null); + const invokable = agent && agent.companyId === companyId && !isAgentAwaitingSetup(agent) + ? await isAgentInvokable(agent) : false; + return { agent, invokable }; + } + + async function hasExternalReviewWait(issue: RecoveryIssue, pass?: RecoveryPassReads) { + if (pass?.issueIds.has(issue.id)) { + const key = `external-review-waits:${issue.companyId}`; + let waiting = pass.reads.get(key) as Promise> | undefined; + if (!waiting) { + waiting = listExternalReviewWaits(db, issue.companyId, [...pass.issueIds]) + .then((waits) => new Set(waits.map((wait) => wait.issueId))); + pass.reads.set(key, waiting); + } + if ((await waiting).has(issue.id)) return true; + } + return (await listExternalReviewWaits(db, issue.companyId, [issue.id])).length > 0; + } + async function getLatestIssueRun( companyId: string, issueId: string, @@ -1562,10 +1596,11 @@ export function recoveryService( async function hasPersistedDurableWaitPath( issue: RecoveryIssue, latestRun: LatestIssueRun, + pass?: RecoveryPassReads, ) { if (issue.monitorNextCheckAt) return true; if (issue.status === "in_review" && latestRun?.status === "succeeded" && - (await listExternalReviewWaits(db, issue.companyId, [issue.id])).length > 0) return true; + await hasExternalReviewWait(issue, pass)) return true; if (issue.status === "in_progress" && latestRun?.status === "succeeded" && latestRun.agentId === issue.assigneeAgentId && await hasCommittedNativePlanWait(db, { companyId: issue.companyId, issueId: issue.id, runId: latestRun.id, agentId: latestRun.agentId })) return true; if (issue.status === "in_progress" && latestRun?.status === "succeeded" && latestRun.agentId === issue.assigneeAgentId && @@ -4752,6 +4787,7 @@ export function recoveryService( issueCreatedAtGte?: Date | null; skipExternalConversationSettlement?: boolean; skipRelatedRecovery?: boolean; + pass?: RecoveryPassReads; }) { const candidates = await db .select(recoveryIssueFields) @@ -4801,6 +4837,8 @@ export function recoveryService( issueIds: [] as string[], }; + const pass = opts?.pass ?? (candidates.length > 1 + ? createRecoveryPassReads(candidates.filter((issue) => issue.status === "in_review").map((issue) => issue.id)) : undefined); const handledIssueIds = await resumeShutdownStoppedRetryPreparations(candidates, result, opts?.onDeferredDeadline); const candidateIssueIds = candidates.map((issue) => issue.id); @@ -4955,15 +4993,11 @@ export function recoveryService( } } - const agent = await getAgent(agentId); + const { agent, invokable: agentInvokable } = await readStrandedAgent(agentId, issue.companyId); if (agent?.companyId === issue.companyId && isAgentAwaitingSetup(agent)) { result.skipped += 1; continue; } - const agentInvokable = - agent && agent.companyId === issue.companyId - ? await isAgentInvokable(agent) - : false; if ( agent?.status === "paused" && agent.companyId === issue.companyId && @@ -5204,7 +5238,7 @@ export function recoveryService( } if ( latestRun?.status === "succeeded" && - (await hasPersistedDurableWaitPath(issue, latestRun)) + (await hasPersistedDurableWaitPath(issue, latestRun, pass)) ) { result.skipped += 1; continue; diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index fc7b39b628b..6a8115737ed 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -8,9 +8,11 @@ import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, issueRecoveryBacks import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; import { workSignalDebug } from "@paperclipai/db/work-signal-debug"; import { recoveryActionSweepCondition } from "./blocker-stall-wakeups.js"; +import { createRecoveryPassReads, type RecoveryPassReads } from "./pass-reads.js"; type Entry = { id: string; companyId: string; dueAt: number; fromDeadline?: true }; -type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number, detail?: { unchanged: boolean }) => void }; +type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number, detail?: { unchanged: boolean }) => void; + pass?: RecoveryPassReads }; const PAGE_SIZE = 100; export const resolvedDependencyCandidate = sql`exists (select 1 from ${issueRelations} r where r.company_id = ${issues.companyId} @@ -81,11 +83,13 @@ export function registerIssueRecoveryWork( const cursors = new Map(); const passes = fullPasses(); let enabled = false; + let pass = createRecoveryPassReads(); const worker: ReturnType> = registerScopedDeadlineWork(coordinator, { queue, reconstructionIntervalMs, backstop: passes.backstop, beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, + beforeChecks: entries => { pass = createRecoveryPassReads(entries.map(entry => entry.id)); }, decodeScope: decodeIssueRecoveryHint, matches: issueRecoveryScopeMatches, load: async scope => { @@ -115,7 +119,7 @@ export function registerIssueRecoveryWork( if (workSignalDebug.enabled) workSignalDebug.event(queue, "check", { issueId: entry.id, companyId: entry.companyId, wake: entry.fromDeadline ? "deadline" : "load", dueAt: entry.dueAt }); let next: number | null = null; - await check({ companyId: entry.companyId, issueId: entry.id, + await check({ companyId: entry.companyId, issueId: entry.id, pass, onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); return next === null ? null : { ...entry, dueAt: next, ...(workSignalDebug.enabled ? { fromDeadline: true as const } : {}) }; }, diff --git a/server/src/services/scoped-deadline-work.ts b/server/src/services/scoped-deadline-work.ts index b6ad70881c6..232b816e484 100644 --- a/server/src/services/scoped-deadline-work.ts +++ b/server/src/services/scoped-deadline-work.ts @@ -28,6 +28,7 @@ export function registerScopedDeadlineWork( reconstructionIntervalMs?: number | ((scope: Scope | undefined) => number); backstop?(scope: Scope | undefined): number | null; beforePass?(): Promise; + beforeChecks?(entries: readonly Entry[]): void; decodeScope(hint: string): Scope | undefined; matches(entry: Entry, scope?: Scope): boolean; /** @@ -197,6 +198,7 @@ export function registerScopedDeadlineWork( .sort((a, b) => (options.priority?.(b) ?? 0) - (options.priority?.(a) ?? 0) || a.dueAt - b.dueAt || a.id.localeCompare(b.id)); const batch = takeCompanyWorkBatch(due, options.batchSize ?? 25, lastCompany); lastCompany = batch.lastCompany; + options.beforeChecks?.(batch.entries); // An owner may share more than one resource (for example a task and an // endpoint). Merge intersecting keys before dispatch so waiting siblings // do not occupy the bounded check slots ahead of independent work.