diff --git a/server/scripts/benchmark-review-attention.ts b/server/scripts/benchmark-review-attention.ts new file mode 100644 index 00000000000..f8af18a08c1 --- /dev/null +++ b/server/scripts/benchmark-review-attention.ts @@ -0,0 +1,310 @@ +/** + * Local reproduction of the two slowest board requests of a company with many + * tasks in review, measured through the real routes on a real PostgreSQL: + * + * GET /api/companies/:companyId/attention/summary (the sidebar count) + * GET /api/companies/:companyId/issues (the dashboard task list) + * + * For each request and each fixture size the script reports the statements + * sent, the server time, the process CPU time, the time the main thread was + * busy, the bytes allocated, and where a CPU profile of the handler puts the + * time. With --baseline-ref it runs the pre-fix modules in a second process + * against the same database and fails unless both versions return the same + * summary counts, the same attention feed, the same task list and the same + * review attention for every task in review. + * + * pnpm --filter @paperclipai/server exec tsx scripts/benchmark-review-attention.ts \ + * --baseline-ref= --tasks=3000,9000 --agents=60 + * + * Options: --tasks (comma-separated fixture sizes, default 3000,9000), + * --agents (default 60), --rounds (default 3: each version runs in a new + * process once per round, in alternating order), --samples (requests measured + * per round, default 5), --cpu-profile-dir (keep the .cpuprofile files). + */ +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { promises as fs } from "node:fs"; +import type { AddressInfo } from "node:net"; +import { availableParallelism, loadavg } from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { getHeapStatistics } from "node:v8"; +import { Session } from "node:inspector/promises"; +import express from "express"; +import { and, desc, eq } from "drizzle-orm"; +import { createDb, issues } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "../src/__tests__/helpers/embedded-postgres.js"; +import { seedReviewAttentionFixture } from "../src/__tests__/helpers/review-attention-fixture.js"; + +const arg = (name: string) => process.argv.find(value => value.startsWith(`--${name}=`))?.slice(name.length + 3); +const root = fileURLToPath(new URL("../../", import.meta.url)); +const scriptPath = fileURLToPath(import.meta.url); +const git = (...args: string[]) => execFileSync("git", args, { cwd: root, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); +// The modules of the fix. The baseline process loads their pre-fix source; +// every other module is the current tree in both processes. +const modules = ["server/src/services/issues.ts", "server/src/services/recovery/issue-graph-liveness.ts"]; +// Functions of the review-attention read path, reported from the CPU profile +// with the time spent in them and everything they call. +const profiledFunctions = ["listIssueReviewAttentionMap", "classifyIssueGraphLiveness", "classifyIssueReviewPaths", + "reviewFinding", "ownerCandidatesForRecoveryIssue", "orderedInvokableAgents", "isInvokableAgent", + "getAgentWorkEligibility", "getAgentOrgChainHealth", "listIssueBlockerAttentionMap", "listIssueBlockedInboxAttentionMap", + "filterIssuesForActor", "enrichAttentionItems"]; +const round = (value: number, digits = 1) => Number(value.toFixed(digits)); +const median = (values: number[]) => [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]!; +// Bytes this process has allocated since it started, also the collected ones. +// A Node version that does not report it gives null in the allocation column. +const allocatedBytes = () => (getHeapStatistics() as { total_allocated_bytes?: number }).total_allocated_bytes ?? Number.NaN; + +type ProfileNode = { id: number; callFrame: { functionName: string; url: string; lineNumber: number }; children?: number[] }; +type Profile = { nodes: ProfileNode[]; samples: number[]; timeDeltas: number[] }; + +function summarizeProfile(profile: Profile, requests: number) { + const self = new Map(); + for (const [index, id] of profile.samples.entries()) self.set(id, (self.get(id) ?? 0) + profile.timeDeltas[index]! / 1_000); + const nodes = new Map(profile.nodes.map(node => [node.id, node])); + const total = new Map(); + const totalOf = (id: number): number => { + const node = nodes.get(id)!; + const value = (self.get(id) ?? 0) + (node.children ?? []).reduce((sum, child) => sum + totalOf(child), 0); + total.set(id, value); + return value; + }; + const rootNode = profile.nodes[0]!; + totalOf(rootNode.id); + // Time under a function counts its outermost frames only. + const inclusive = new Map(); + const walk = (id: number, open: Set) => { + const node = nodes.get(id)!; + const name = node.callFrame.functionName; + const counted = profiledFunctions.includes(name) && !open.has(name); + if (counted) inclusive.set(name, (inclusive.get(name) ?? 0) + total.get(id)!); + const next = counted ? new Set(open).add(name) : open; + for (const child of node.children ?? []) walk(child, next); + }; + walk(rootNode.id, new Set()); + const selfByFunction = new Map(); + for (const node of profile.nodes) { + const file = node.callFrame.url ? `${path.basename(node.callFrame.url)}:${node.callFrame.lineNumber + 1}` : ""; + const key = `${node.callFrame.functionName || "(anonymous)"}${file ? ` ${file}` : ""}`; + selfByFunction.set(key, (selfByFunction.get(key) ?? 0) + (self.get(node.id) ?? 0)); + } + // The profiler's own stop call is not part of a request. + const excluded = (key: string) => key.startsWith("(idle)") || key.startsWith("post node:inspector"); + const idle = [...selfByFunction].filter(([key]) => excluded(key)).reduce((sum, [, ms]) => sum + ms, 0); + const busy = [...selfByFunction].reduce((sum, [, ms]) => sum + ms, 0) - idle; + const perRequest = (ms: number) => round(ms / requests); + return { + busyMsPerRequest: perRequest(busy), + garbageCollectorMsPerRequest: perRequest(selfByFunction.get("(garbage collector)") ?? 0), + inclusiveMsPerRequest: Object.fromEntries([...inclusive].sort((a, b) => b[1] - a[1]) + .map(([name, ms]) => [name, { ms: perRequest(ms), shareOfBusy: round(ms / busy * 100) }])), + topSelfMsPerRequest: Object.fromEntries([...selfByFunction].filter(([key]) => !excluded(key)) + .sort((a, b) => b[1] - a[1]).slice(0, 12).map(([key, ms]) => [key, perRequest(ms)])), + }; +} + +async function runArm() { + const { attentionRoutes } = await import("../src/routes/attention.js"); + const { issueRoutes } = await import("../src/routes/issues.js"); + const { errorHandler } = await import("../src/middleware/index.js"); + const { issueService } = await import("../src/services/issues.js"); + const companyId = arg("company")!, userId = arg("user")!, samples = Number(arg("samples") ?? 5); + const db = createDb(arg("database-url")!); + const client = db.$client as unknown as { unsafe: (query: string, ...rest: unknown[]) => unknown }; + const unsafe = client.unsafe; + let statements = 0, sqlBytes = 0; + client.unsafe = function (this: unknown, query: string, ...rest: unknown[]) { + statements += 1; + sqlBytes += Buffer.byteLength(query); + return unsafe.call(this, query, ...rest); + }; + let serverMs = 0; + const app = express(); + app.use((req, res, next) => { + const started = performance.now(); + res.on("finish", () => { serverMs = performance.now() - started; }); + (req as unknown as { actor: unknown }).actor = { type: "board", source: "session", userId, companyIds: [companyId], isInstanceAdmin: false }; + next(); + }); + app.use("/api", attentionRoutes(db)); + app.use("/api", issueRoutes(db, {} as never)); + app.use(errorHandler); + const server = app.listen(0, "127.0.0.1"); + await new Promise(resolve => server.once("listening", resolve)); + const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}/api/companies/${companyId}`; + const get = async (route: string) => { + const response = await fetch(`${base}${route}`); + const body = Buffer.from(await response.arrayBuffer()); + assert.equal(response.status, 200, `${route}: ${body.toString("utf8").slice(0, 400)}`); + return body; + }; + const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex"); + const requests = { summary: "/attention/summary", taskList: "/issues" }; + const results: Record = {}; + try { + for (const [name, route] of Object.entries(requests)) { + let bytes = 0; + for (let warm = 0; warm < 2; warm++) bytes = (await get(route)).length; + const measured: Array> = []; + for (let sample = 0; sample < samples; sample++) { + globalThis.gc?.(); + statements = 0; sqlBytes = 0; + const cpu = process.cpuUsage(), loop = performance.eventLoopUtilization(); + const allocated = allocatedBytes(), started = performance.now(); + await get(route); + const used = process.cpuUsage(cpu); + measured.push({ wallMs: performance.now() - started, serverMs, cpuMs: (used.user + used.system) / 1_000, + mainThreadBusyMs: performance.eventLoopUtilization(loop).active, + allocatedMb: (allocatedBytes() - allocated) / 1_048_576, statements, sqlBytes }); + } + const profiler = new Session(); + profiler.connect(); + await profiler.post("Profiler.enable"); + await profiler.post("Profiler.setSamplingInterval", { interval: 200 }); + await profiler.post("Profiler.start"); + const profiledRequests = 3; + for (let request = 0; request < profiledRequests; request++) await get(route); + const { profile } = await profiler.post("Profiler.stop"); + profiler.disconnect(); + if (arg("cpu-profile")) await fs.writeFile(`${arg("cpu-profile")}.${name}.cpuprofile`, JSON.stringify(profile)); + results[name] = { route, responseBytes: bytes, measured, profile: summarizeProfile(profile as unknown as Profile, profiledRequests) }; + } + // What both versions must agree on. + const summary = JSON.parse((await get(requests.summary)).toString("utf8")); + const feed = JSON.parse((await get("/attention?all=true")).toString("utf8")); + const taskList = JSON.parse((await get(requests.taskList)).toString("utf8")) as Array<{ status: string; reviewAttention?: { state: string } }>; + const reviewRows = await db.select().from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.status, "in_review"))).orderBy(desc(issues.id)); + const reviewAttention = [...await issueService(db).listReviewAttention(companyId, reviewRows)]; + const states: Record = {}; + for (const [, attention] of reviewAttention) states[attention.state] = (states[attention.state] ?? 0) + 1; + console.log(`RESULT ${JSON.stringify({ + results, + shape: { totalCount: summary.totalCount, countsBySourceKind: summary.countsBySourceKind, feedItems: feed.items.length, + taskListRows: taskList.length, taskListReviews: taskList.filter(row => row.status === "in_review").length, + reviewAttentionStates: states }, + digests: { summary: digest({ ...summary, generatedAt: null }), feed: digest({ ...feed, generatedAt: null }), + taskList: digest(taskList), reviewAttention: digest(reviewAttention) }, + })}`); + } finally { + await new Promise(resolve => server.close(resolve)); + await db.$client.end({ timeout: 0 }); + } +} + +async function main() { + const sizes = (arg("tasks") ?? "3000,9000").split(",").map(Number), agentCount = Number(arg("agents") ?? 60); + const samples = Number(arg("samples") ?? 5), rounds = Number(arg("rounds") ?? 3); + for (const value of [...sizes, agentCount, samples, rounds]) assert.ok(Number.isSafeInteger(value) && value > 0, "sizes must be positive integers"); + const baselineRef = arg("baseline-ref"); + const baselineRevision = baselineRef ? git("rev-parse", "--verify", `${baselineRef}^{commit}`).trim() : null; + let baselineDir: string | undefined; + const report: Record = {}; + try { + if (baselineRevision) { + baselineDir = await fs.mkdtemp(path.join(root, "server/scripts/.review-attention-baseline-")); + const redirects: Record = {}; + for (const file of modules) { + // Relative imports of a pre-fix module resolve to the current tree; + // the hook below sends the ones that name a module of the fix to its + // pre-fix copy. + const source = git("show", `${baselineRevision}:${file}`).replace(/from "(\.{1,2}\/[^"]+)"/g, + (_match, specifier: string) => `from ${JSON.stringify(fileURLToPath(new URL(specifier, pathToFileURL(path.join(root, file)))))}`); + const copy = path.join(baselineDir, file.replaceAll("/", "__")); + await fs.writeFile(copy, source); + redirects[pathToFileURL(path.join(root, file)).href] = pathToFileURL(copy).href; + } + await fs.writeFile(path.join(baselineDir, "hooks.mjs"), `let redirects = {}; +export function initialize(data) { redirects = data; } +export async function resolve(specifier, context, nextResolve) { + const resolved = await nextResolve(specifier, context); + return redirects[resolved.url] ? { ...resolved, url: redirects[resolved.url] } : resolved; +} +`); + await fs.writeFile(path.join(baselineDir, "register.mjs"), `import { register } from "node:module"; +register(new URL("./hooks.mjs", import.meta.url), { data: ${JSON.stringify(redirects)} }); +`); + } + const arms = baselineDir ? ["baseline", "treatment"] : ["treatment"]; + for (const [sizeIndex, tasks] of sizes.entries()) { + const temporary = await startEmbeddedPostgresTestDatabase("review-attention-benchmark-"); + try { + const db = createDb(temporary.connectionString); + const seedStarted = performance.now(); + // A second company with the same shapes, which no request may read. + await seedReviewAttentionFixture(db, { tasks: Math.ceil(tasks / 5), agents: agentCount, prefix: "OTH" }); + const fixture = await seedReviewAttentionFixture(db, { tasks, agents: agentCount, prefix: "SYN" }); + await db.$client.unsafe("vacuum analyze"); + await db.$client.end({ timeout: 0 }); + const seedMs = Math.round(performance.now() - seedStarted); + type ArmRequest = { route: string; responseBytes: number; measured: Array>; profile: ReturnType }; + type ArmOutput = { results: Record; shape: unknown; digests: Record }; + const byArm: Record = {}; + // Each version runs once per round, in an order that alternates + // between rounds and sizes, so load on the host reaches both. + for (let turn = 0; turn < rounds; turn++) for (const arm of (sizeIndex + turn) % 2 === 0 ? arms : [...arms].reverse()) { + const output = await new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["--expose-gc", "--import", "tsx", + ...(arm === "baseline" ? ["--import", pathToFileURL(path.join(baselineDir!, "register.mjs")).href] : []), + scriptPath, `--arm=${arm}`, `--database-url=${temporary.connectionString}`, `--company=${fixture.companyId}`, + `--user=${fixture.userId}`, `--samples=${samples}`, + ...(arg("cpu-profile-dir") ? [`--cpu-profile=${path.join(arg("cpu-profile-dir")!, `${arm}-${tasks}`)}`] : [])], + { cwd: path.join(root, "server"), stdio: ["ignore", "pipe", "inherit"], env: { ...process.env, NODE_ENV: "production" } }); + let stdout = ""; + child.stdout.on("data", chunk => { stdout += chunk; }); + child.on("error", reject); + child.on("exit", code => code === 0 ? resolve(stdout) : reject(new Error(`${arm} exited with ${code}\n${stdout.slice(-2000)}`))); + }); + const line = output.split("\n").find(value => value.startsWith("RESULT ")); + assert.ok(line, `${arm} printed no result`); + (byArm[arm] ??= []).push(JSON.parse(line.slice(7))); + } + const first = byArm.treatment![0]!; + for (const output of Object.values(byArm).flat()) { + assert.deepEqual(output.shape, first.shape, "Every run of both versions must return the same counts"); + assert.deepEqual(output.digests, first.digests, + "Every run of both versions must return the same summary, feed, task list and review attention"); + } + // The samples of every round of a version, and the profile of its last round. + const summarize = (outputs: ArmOutput[]) => Object.fromEntries(Object.keys(first.results).map(name => { + const measured = outputs.flatMap(output => output.results[name]!.measured); + const field = (key: string) => measured.map(sample => sample[key]!); + const last = outputs.at(-1)!.results[name]!; + return [name, { + route: last.route, responseBytes: last.responseBytes, statements: median(field("statements")), + sqlTextBytes: median(field("sqlBytes")), + serverMs: { median: round(median(field("serverMs"))), min: round(Math.min(...field("serverMs"))) }, + cpuMs: { median: round(median(field("cpuMs"))), min: round(Math.min(...field("cpuMs"))) }, + mainThreadBusyMs: { median: round(median(field("mainThreadBusyMs"))), min: round(Math.min(...field("mainThreadBusyMs"))) }, + allocatedMb: round(median(field("allocatedMb"))), profile: last.profile, + }]; + })); + report[`tasks=${tasks}`] = { + fixture: { synthetic: true, ...fixture.counts, otherCompanyTasks: Math.ceil(tasks / 5), seedMs }, + shape: first.shape, + identicalResponses: byArm.baseline ? Object.keys(first.digests) : "no baseline", + ...Object.fromEntries(Object.entries(byArm).map(([arm, outputs]) => [arm, summarize(outputs)])), + }; + } finally { + await temporary.cleanup(); + } + } + console.log(JSON.stringify({ + baselineRevision, treatmentRevision: git("rev-parse", "HEAD").trim(), + treatmentDirty: git("status", "--porcelain", "--", ...modules).trim().length > 0, + node: process.version, host: { availableCpus: availableParallelism(), loadAverage: loadavg().map(value => round(value, 2)) }, + rounds, samplesPerRound: samples, warmupsPerRound: 2, report, + }, null, 2)); + } finally { + if (baselineDir) await fs.rm(baselineDir, { recursive: true, force: true }); + } +} + +const keepAlive = setInterval(() => {}, 1_000); +try { + await (arg("arm") ? runArm() : main()); +} finally { + clearInterval(keepAlive); +} diff --git a/server/src/__tests__/helpers/issue-graph-liveness-reference.ts b/server/src/__tests__/helpers/issue-graph-liveness-reference.ts new file mode 100644 index 00000000000..3e88b0c8a99 --- /dev/null +++ b/server/src/__tests__/helpers/issue-graph-liveness-reference.ts @@ -0,0 +1,642 @@ +/** + * The task-graph liveness classification as it was before its agents and paths + * were indexed once per classification. It is the oracle of the equivalence + * tests: the classification under `services/recovery` must return exactly what + * this returns for every input. + * + * It is slow on purpose. Every check rebuilds the agent list and scans every + * path, which is the behaviour the tests measure the current code against. Do + * not optimise it. When a classification rule changes on purpose, make the same + * change here. + */ +import { getAgentWorkEligibility, isAgentInvokable } from "@paperclipai/shared"; +import { buildIssueGraphLivenessIncidentKey } from "../../services/recovery/origins.js"; +import type { + IssueGraphLivenessInput, + IssueLivenessAgentInput, + IssueLivenessDependencyPathEntry, + IssueLivenessExecutionPathInput, + IssueLivenessFinding, + IssueLivenessIssueInput, + IssueLivenessOwnerCandidate, + IssueLivenessOwnerCandidateReason, + IssueLivenessRelationInput, + IssueLivenessSeverity, + IssueLivenessState, + IssueLivenessWaitingPathInput, + IssueReviewPathFact, +} from "../../services/recovery/issue-graph-liveness.js"; + +function issueLabel(issue: IssueLivenessIssueInput) { + return issue.identifier ?? issue.id; +} + +function pathEntry(issue: IssueLivenessIssueInput): IssueLivenessDependencyPathEntry { + return { + issueId: issue.id, + identifier: issue.identifier, + title: issue.title, + status: issue.status, + }; +} + +function isInvokableAgent( + agent: IssueLivenessAgentInput | null | undefined, + agentsById: Map, +) { + return Boolean(agent && isAgentInvokable({ agent, agents: [...agentsById.values()] })); +} + +function hasActiveExecutionPath( + companyId: string, + issueId: string, + activeRuns: IssueLivenessExecutionPathInput[], + queuedWakeRequests: IssueLivenessExecutionPathInput[], +) { + return [...activeRuns, ...queuedWakeRequests].some( + (entry) => entry.companyId === companyId && entry.issueId === issueId, + ); +} + +function hasWaitingPath( + companyId: string, + issueId: string, + waitingPaths: IssueLivenessWaitingPathInput[], +) { + return waitingPaths.some((entry) => entry.companyId === companyId && entry.issueId === issueId); +} + +function readRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? value as Record + : null; +} + +function readPositiveInteger(value: unknown): number | null { + return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : null; +} + +function readDateMs(value: unknown): number | null { + if (!(typeof value === "string" || value instanceof Date)) return null; + const date = value instanceof Date ? value : new Date(value); + const time = date.getTime(); + return Number.isNaN(time) ? null : time; +} + +function monitorFromIssue(issue: IssueLivenessIssueInput) { + const policyMonitor = readRecord(readRecord(issue.executionPolicy)?.monitor); + const stateMonitor = readRecord(readRecord(issue.executionState)?.monitor); + return { policyMonitor, stateMonitor }; +} + +function hasUnconsumedIssueMonitorPath(issue: IssueLivenessIssueInput, now: Date | string | number) { + const nowMs = typeof now === "number" ? now : readDateMs(now) ?? Date.now(); + const nextCheckAtMs = readDateMs(issue.monitorNextCheckAt); + if (nextCheckAtMs === null) return false; + + const { policyMonitor, stateMonitor } = monitorFromIssue(issue); + const timeoutAtMs = readDateMs(policyMonitor?.timeoutAt ?? stateMonitor?.timeoutAt); + if (timeoutAtMs !== null && timeoutAtMs <= nowMs) return false; + + const maxAttempts = readPositiveInteger(policyMonitor?.maxAttempts ?? stateMonitor?.maxAttempts); + const stateAttemptCount = readPositiveInteger(stateMonitor?.attemptCount) ?? 0; + const attemptCount = issue.monitorAttemptCount ?? stateAttemptCount; + if (maxAttempts !== null && attemptCount >= maxAttempts) return false; + + return true; +} + +function hasScheduledIssueMonitorPath(issue: IssueLivenessIssueInput, now: Date | string | number) { + const nowMs = typeof now === "number" ? now : readDateMs(now) ?? Date.now(); + const nextCheckAtMs = readDateMs(issue.monitorNextCheckAt); + return nextCheckAtMs !== null && nextCheckAtMs > nowMs && hasUnconsumedIssueMonitorPath(issue, nowMs); +} + +export function referenceClassifyIssueReviewPaths( + input: IssueGraphLivenessInput, + issue: IssueLivenessIssueInput, +): IssueReviewPathFact[] { + if (issue.status !== "in_review") return []; + const nowMs = readDateMs(input.now ?? new Date()) ?? Date.now(); + const agentsById = new Map(input.agents.map((agent) => [agent.id, agent])); + const paths: IssueReviewPathFact[] = []; + if (issue.conversationAgentId && issue.conversationUserId && issue.conversationState === "waiting") { + return [{ kind: "human_reviewer", ref: issue.conversationUserId, userId: issue.conversationUserId, agentId: null, since: null }]; + } + + if (issue.assigneeUserId) { + paths.push({ + kind: "human_reviewer", + ref: issue.assigneeUserId, + agentId: null, + userId: issue.assigneeUserId, + since: null, + }); + } + + const participant = issue.executionState?.status === "pending" + ? issue.executionState.currentParticipant + : null; + const participantAgentId = readPrincipalAgentId(participant); + if (participantAgentId) { + const participantAgent = agentsById.get(participantAgentId); + if (participantAgent?.companyId === issue.companyId && isInvokableAgent(participantAgent, agentsById)) { + paths.push({ + kind: "execution_participant", + ref: participantAgentId, + agentId: participantAgentId, + userId: null, + since: null, + }); + } + } else if (principalIsResolvableUser(participant)) { + const userId = (participant as Record).userId as string; + paths.push({ + kind: "execution_participant", + ref: userId, + agentId: null, + userId, + since: null, + }); + } + + if (hasScheduledIssueMonitorPath(issue, nowMs)) { + paths.push({ kind: "monitor", ref: null, agentId: issue.assigneeAgentId ?? null, userId: null, since: null }); + } + + const appendExecutionPaths = ( + entries: IssueLivenessExecutionPathInput[], + kind: "active_run" | "queued_wake", + ) => { + for (const entry of entries) { + if (entry.companyId !== issue.companyId || entry.issueId !== issue.id) continue; + paths.push({ + kind, + ref: entry.id ?? null, + agentId: entry.agentId ?? null, + userId: null, + since: entry.createdAt ?? null, + }); + } + }; + appendExecutionPaths(input.activeRuns ?? [], "active_run"); + appendExecutionPaths(input.queuedWakeRequests ?? [], "queued_wake"); + + const appendWaitingPaths = ( + entries: IssueLivenessWaitingPathInput[], + kind: "interaction" | "approval" | "recovery" | "external_response", + ) => { + for (const entry of entries) { + if (entry.companyId !== issue.companyId || entry.issueId !== issue.id) continue; + paths.push({ + kind, + ref: entry.id ?? null, + agentId: null, + userId: null, + since: entry.createdAt ?? null, + }); + } + }; + appendWaitingPaths(input.pendingInteractions ?? [], "interaction"); + appendWaitingPaths(input.pendingApprovals ?? [], "approval"); + appendWaitingPaths(input.externalResponseWaits ?? [], "external_response"); + appendWaitingPaths(input.openRecoveryIssues ?? [], "recovery"); + + return paths; +} + +function readPrincipalAgentId(principal: unknown): string | null { + if (!principal || typeof principal !== "object") return null; + const value = principal as Record; + return value.type === "agent" && typeof value.agentId === "string" && value.agentId.length > 0 + ? value.agentId + : null; +} + +function principalIsResolvableUser(principal: unknown): boolean { + if (!principal || typeof principal !== "object") return false; + const value = principal as Record; + return value.type === "user" && typeof value.userId === "string" && value.userId.length > 0; +} + +function addOwnerCandidate( + candidates: IssueLivenessOwnerCandidate[], + seen: Set, + agentsById: Map, + companyId: string, + agentId: string | null | undefined, + reason: IssueLivenessOwnerCandidateReason, + sourceIssueId: string, +) { + if (!agentId || seen.has(agentId)) return; + const agent = agentsById.get(agentId); + if (!agent || agent.companyId !== companyId || !isInvokableAgent(agent, agentsById)) return; + seen.add(agentId); + candidates.push({ agentId, reason, sourceIssueId }); +} + +function addAgentChainCandidates( + candidates: IssueLivenessOwnerCandidate[], + seen: Set, + startAgentId: string | null | undefined, + agentsById: Map, + companyId: string, + reason: IssueLivenessOwnerCandidateReason, + sourceIssueId: string, +) { + const chainSeen = new Set(); + let current = startAgentId ? agentsById.get(startAgentId) : null; + + while (current?.reportsTo) { + if (chainSeen.has(current.reportsTo)) break; + chainSeen.add(current.reportsTo); + const manager = agentsById.get(current.reportsTo); + if (!manager || manager.companyId !== companyId) break; + addOwnerCandidate(candidates, seen, agentsById, companyId, manager.id, reason, sourceIssueId); + current = manager; + } +} + +function orderedInvokableAgents( + agents: IssueLivenessAgentInput[], + agentsById: Map, + companyId: string, +) { + return agents + .filter((agent) => agent.companyId === companyId && isInvokableAgent(agent, agentsById)) + .sort((left, right) => left.id.localeCompare(right.id)); +} + +function ownerCandidatesForRecoveryIssue( + issue: IssueLivenessIssueInput, + agents: IssueLivenessAgentInput[], + agentsById: Map, + options: { + includeStalledAssignee?: boolean; + } = {}, +) { + const candidates: IssueLivenessOwnerCandidate[] = []; + const seen = new Set(); + + if (options.includeStalledAssignee && issue.status !== "cancelled" && issue.status !== "done") { + addOwnerCandidate( + candidates, + seen, + agentsById, + issue.companyId, + issue.assigneeAgentId, + "stalled_blocker_assignee", + issue.id, + ); + } + + addAgentChainCandidates( + candidates, + seen, + issue.assigneeAgentId, + agentsById, + issue.companyId, + "assignee_reporting_chain", + issue.id, + ); + addAgentChainCandidates( + candidates, + seen, + issue.createdByAgentId, + agentsById, + issue.companyId, + "creator_reporting_chain", + issue.id, + ); + + const invokableAgents = orderedInvokableAgents(agents, agentsById, issue.companyId); + for (const agent of invokableAgents) { + if (!agent.reportsTo) { + addOwnerCandidate(candidates, seen, agentsById, issue.companyId, agent.id, "root_agent", issue.id); + } + } + for (const agent of invokableAgents) { + addOwnerCandidate( + candidates, + seen, + agentsById, + issue.companyId, + agent.id, + "ordered_invokable_fallback", + issue.id, + ); + } + + return candidates; +} + +function incidentKey(input: { + companyId: string; + issueId: string; + state: IssueLivenessState; + blockerIssueId?: string | null; + participantAgentId?: string | null; +}) { + return buildIssueGraphLivenessIncidentKey(input); +} + +function finding(input: { + issue: IssueLivenessIssueInput; + state: IssueLivenessState; + severity?: IssueLivenessSeverity; + reason: string; + dependencyPath: IssueLivenessIssueInput[]; + recoveryIssue: IssueLivenessIssueInput; + recommendedOwnerCandidateAgentIds: string[]; + recommendedOwnerCandidates: IssueLivenessOwnerCandidate[]; + recommendedAction: string; + blockerIssueId?: string | null; + participantAgentId?: string | null; +}): IssueLivenessFinding { + return { + issueId: input.issue.id, + companyId: input.issue.companyId, + identifier: input.issue.identifier, + state: input.state, + severity: input.severity ?? "critical", + reason: input.reason, + dependencyPath: input.dependencyPath.map(pathEntry), + recoveryIssueId: input.recoveryIssue.id, + recommendedOwnerAgentId: input.recommendedOwnerCandidateAgentIds[0] ?? null, + recommendedOwnerCandidateAgentIds: input.recommendedOwnerCandidateAgentIds, + recommendedOwnerCandidates: input.recommendedOwnerCandidates, + recommendedAction: input.recommendedAction, + incidentKey: incidentKey({ + companyId: input.issue.companyId, + issueId: input.issue.id, + state: input.state, + blockerIssueId: input.blockerIssueId, + participantAgentId: input.participantAgentId, + }), + }; +} + +export function referenceClassifyIssueGraphLiveness(input: IssueGraphLivenessInput): IssueLivenessFinding[] { + const nowMs = readDateMs(input.now ?? new Date()) ?? Date.now(); + const issuesById = new Map(input.issues.map((issue) => [issue.id, issue])); + const agentsById = new Map(input.agents.map((agent) => [agent.id, agent])); + const blockersByBlockedIssueId = new Map(); + const unresolvedBlockers = new Set(); + const findings: IssueLivenessFinding[] = []; + const activeRuns = input.activeRuns ?? []; + const queuedWakeRequests = input.queuedWakeRequests ?? []; + const pendingInteractions = input.pendingInteractions ?? []; + const pendingApprovals = input.pendingApprovals ?? []; + const openRecoveryIssues = input.openRecoveryIssues ?? []; + + for (const relation of input.relations) { + const list = blockersByBlockedIssueId.get(relation.blockedIssueId) ?? []; + list.push(relation); + blockersByBlockedIssueId.set(relation.blockedIssueId, list); + + const blocker = issuesById.get(relation.blockerIssueId); + const blocked = issuesById.get(relation.blockedIssueId); + if ( + blocker && + blocked && + blocker.companyId === relation.companyId && + blocked.companyId === relation.companyId && + blocker.status !== "done" && + blocker.status !== "cancelled" && + blocked.status === "blocked" + ) { + unresolvedBlockers.add(blocker.id); + } + } + + for (const relations of blockersByBlockedIssueId.values()) { + relations.sort((left, right) => { + const leftIssue = issuesById.get(left.blockerIssueId); + const rightIssue = issuesById.get(right.blockerIssueId); + const leftLabel = leftIssue ? issueLabel(leftIssue) : left.blockerIssueId; + const rightLabel = rightIssue ? issueLabel(rightIssue) : right.blockerIssueId; + return leftLabel.localeCompare(rightLabel); + }); + } + + function hasExplicitWaitingPath(issue: IssueLivenessIssueInput) { + return Boolean(issue.assigneeUserId) || + hasScheduledIssueMonitorPath(issue, nowMs) || + hasActiveExecutionPath(issue.companyId, issue.id, activeRuns, queuedWakeRequests) || + hasWaitingPath(issue.companyId, issue.id, pendingInteractions) || + hasWaitingPath(issue.companyId, issue.id, pendingApprovals) || + hasWaitingPath(issue.companyId, issue.id, openRecoveryIssues); + } + + function reviewFinding( + source: IssueLivenessIssueInput, + reviewIssue: IssueLivenessIssueInput, + dependencyPath: IssueLivenessIssueInput[], + ): IssueLivenessFinding | null { + if (reviewIssue.status !== "in_review") return null; + if (referenceClassifyIssueReviewPaths(input, reviewIssue).length > 0) return null; + + const ownerCandidates = ownerCandidatesForRecoveryIssue(reviewIssue, input.agents, agentsById, { + includeStalledAssignee: true, + }); + + const hasPendingExecutionState = reviewIssue.executionState?.status === "pending"; + const participant = hasPendingExecutionState + ? reviewIssue.executionState?.currentParticipant + : null; + const participantAgentId = readPrincipalAgentId(participant); + if (participantAgentId) { + const participantAgent = agentsById.get(participantAgentId); + if (isInvokableAgent(participantAgent, agentsById) && participantAgent?.companyId === reviewIssue.companyId) return null; + + return finding({ + issue: source, + state: "invalid_review_participant", + reason: participantAgent + ? `${issueLabel(reviewIssue)} is in review, but current participant agent is ${participantAgent.status}.` + : `${issueLabel(reviewIssue)} is in review, but current participant agent cannot be resolved.`, + dependencyPath, + recoveryIssue: reviewIssue, + recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId), + recommendedOwnerCandidates: ownerCandidates, + recommendedAction: + `Repair ${issueLabel(reviewIssue)}'s review participant or return the issue to an active assignee with a clear change request.`, + participantAgentId, + }); + } + + if (principalIsResolvableUser(participant)) return null; + + if (hasPendingExecutionState) { + return finding({ + issue: source, + state: "invalid_review_participant", + reason: `${issueLabel(reviewIssue)} is in review, but its current participant cannot be resolved.`, + dependencyPath, + recoveryIssue: reviewIssue, + recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId), + recommendedOwnerCandidates: ownerCandidates, + recommendedAction: + `Repair ${issueLabel(reviewIssue)}'s review participant or return the issue to an active assignee with a clear change request.`, + }); + } + + if (!reviewIssue.assigneeAgentId || reviewIssue.assigneeUserId) return null; + + return finding({ + issue: source, + state: "in_review_without_action_path", + reason: `${issueLabel(reviewIssue)} is in review with an agent assignee but no participant, interaction, approval, user owner, wake, active run, or recovery issue owning the next action.`, + dependencyPath, + recoveryIssue: reviewIssue, + recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId), + recommendedOwnerCandidates: ownerCandidates, + recommendedAction: + `Review ${issueLabel(reviewIssue)} and make the next action explicit: add a reviewer/interaction, return it to active work with a change request, mark it done if accepted, or open a bounded recovery issue.`, + blockerIssueId: reviewIssue.id, + }); + } + + function blockedFindingForLeaf( + source: IssueLivenessIssueInput, + blocker: IssueLivenessIssueInput, + dependencyPath: IssueLivenessIssueInput[], + ): IssueLivenessFinding | null { + const ownerCandidates = ownerCandidatesForRecoveryIssue(blocker, input.agents, agentsById, { + includeStalledAssignee: true, + }); + + if (blocker.status === "cancelled") { + return finding({ + issue: source, + state: "blocked_by_cancelled_issue", + reason: `${issueLabel(source)} is still blocked by cancelled issue ${issueLabel(blocker)}.`, + dependencyPath, + recoveryIssue: blocker, + recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId), + recommendedOwnerCandidates: ownerCandidates, + recommendedAction: + `Inspect ${issueLabel(blocker)} and either remove it from ${issueLabel(source)}'s blockers or replace it with an actionable unblock issue.`, + blockerIssueId: blocker.id, + }); + } + + if (hasExplicitWaitingPath(blocker)) return null; + + if (blocker.status === "in_review") { + return reviewFinding(source, blocker, dependencyPath); + } + + if (blocker.status === "backlog" && blocker.assigneeAgentId) { + return finding({ + issue: source, + state: "blocked_by_assigned_backlog_issue", + reason: `${issueLabel(source)} is blocked by assigned backlog issue ${issueLabel(blocker)} with no wake, active run, human owner, interaction, approval, monitor, or recovery issue owning the next action.`, + dependencyPath, + recoveryIssue: blocker, + recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId), + recommendedOwnerCandidates: ownerCandidates, + recommendedAction: + `Review ${issueLabel(blocker)} and either move it to todo so the assignee wakes, assign a human owner or interaction if it is intentionally parked, or remove it from ${issueLabel(source)}'s blockers if it is no longer required.`, + blockerIssueId: blocker.id, + }); + } + + if (!blocker.assigneeAgentId && !blocker.assigneeUserId) { + return finding({ + issue: source, + state: "blocked_by_unassigned_issue", + reason: `${issueLabel(source)} is blocked by unassigned issue ${issueLabel(blocker)} with no user owner.`, + dependencyPath, + recoveryIssue: blocker, + recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId), + recommendedOwnerCandidates: ownerCandidates, + recommendedAction: + `Assign ${issueLabel(blocker)} to an owner who can complete it, or remove it from ${issueLabel(source)}'s blockers if it is no longer required.`, + blockerIssueId: blocker.id, + }); + } + + if (!blocker.assigneeAgentId) return null; + + const blockerAgent = agentsById.get(blocker.assigneeAgentId); + const blockerEligibility = blockerAgent + ? getAgentWorkEligibility({ agent: blockerAgent, agents: input.agents }) + : null; + if (!blockerAgent || blockerAgent.companyId !== source.companyId || !blockerEligibility?.invokable) { + return finding({ + issue: source, + state: "blocked_by_uninvokable_assignee", + reason: blockerAgent + ? `${issueLabel(source)} is blocked by ${issueLabel(blocker)}, but its assignee is ${blockerEligibility?.invokabilityReason === "invalid_org_chain" ? "in an invalid org chain" : blockerAgent.status}.` + : `${issueLabel(source)} is blocked by ${issueLabel(blocker)}, but its assignee no longer exists.`, + dependencyPath, + recoveryIssue: blocker, + recommendedOwnerCandidateAgentIds: ownerCandidates.map((candidate) => candidate.agentId), + recommendedOwnerCandidates: ownerCandidates, + recommendedAction: + `Review ${issueLabel(blocker)} and assign it to an active owner or replace the blocker with an actionable issue.`, + blockerIssueId: blocker.id, + }); + } + + return null; + } + + function firstBlockedChainFinding( + source: IssueLivenessIssueInput, + current: IssueLivenessIssueInput, + dependencyPath: IssueLivenessIssueInput[], + seen: Set, + ): IssueLivenessFinding | null { + if (seen.has(current.id)) return null; + seen.add(current.id); + + const relations = blockersByBlockedIssueId.get(current.id) ?? []; + for (const relation of relations) { + if (relation.companyId !== current.companyId || relation.companyId !== source.companyId) continue; + const blocker = issuesById.get(relation.blockerIssueId); + if (!blocker || blocker.companyId !== source.companyId || blocker.status === "done") continue; + const path = [...dependencyPath, blocker]; + + if (blocker.status === "blocked") { + const nested = firstBlockedChainFinding(source, blocker, path, new Set(seen)); + if (nested) return nested; + if (hasExplicitWaitingPath(blocker)) continue; + } + + const leafFinding = blockedFindingForLeaf(source, blocker, path); + if (leafFinding) return leafFinding; + } + + return null; + } + + for (const issue of input.issues) { + const hasUnresolvedBlockerEdge = (blockersByBlockedIssueId.get(issue.id) ?? []).some((relation) => { + if (relation.companyId !== issue.companyId) return false; + const blocker = issuesById.get(relation.blockerIssueId); + return Boolean(blocker && blocker.companyId === issue.companyId && blocker.status !== "done"); + }); + const shouldInspectBlockedChain = issue.status === "blocked" || ( + issue.status !== "done" && + issue.status !== "cancelled" && + Boolean(issue.assigneeAgentId) && + hasUnresolvedBlockerEdge + ); + + let chainFinding: IssueLivenessFinding | null = null; + if (shouldInspectBlockedChain) { + if (unresolvedBlockers.has(issue.id)) continue; + chainFinding = firstBlockedChainFinding(issue, issue, [issue], new Set()); + if (chainFinding) findings.push(chainFinding); + } + + if (issue.status === "in_review" && !chainFinding && !unresolvedBlockers.has(issue.id)) { + const review = reviewFinding(issue, issue, [issue]); + if (review) findings.push(review); + } + } + + return findings; +} diff --git a/server/src/__tests__/helpers/review-attention-fixture.ts b/server/src/__tests__/helpers/review-attention-fixture.ts new file mode 100644 index 00000000000..07d47a11002 --- /dev/null +++ b/server/src/__tests__/helpers/review-attention-fixture.ts @@ -0,0 +1,345 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + agentWakeupRequests, + agents, + approvals, + authUsers, + chatActions, + chatConversations, + chatDeliveries, + chatEndpointResources, + chatEndpoints, + chatExternalPrincipals, + chatGitHubReviews, + chatIdentityLinks, + chatMessageLinks, + companies, + companyMemberships, + heartbeatRuns, + issueApprovals, + issueComments, + issueRecoveryActions, + issueRelations, + issueThreadInteractions, + issues, + runIdentityContexts, + toolApplications, + toolConnections, + type Db, +} from "@paperclipai/db"; +import { defaultGitHubReviewPolicy } from "@paperclipai/shared"; +import { buildIssueGraphLivenessIncidentKey, RECOVERY_ORIGIN_KINDS } from "../../services/recovery/origins.js"; + +/** + * A synthetic company for the review-attention read path: tasks of every + * status with children, tasks in review that cover every review path and every + * way a review stalls, blocked tasks behind chains, cycles and terminal + * blockers, open recovery tasks, and reviews that wait for an external reply. + * The shape depends only on the options, so two fixtures of the same options + * differ only in their ids. + */ +export interface ReviewAttentionFixtureOptions { + tasks: number; + agents: number; + prefix: string; + /** Open recovery tasks whose origin names a task of this company. */ + recoveryTasks?: number; + descriptionBytes?: number; + /** Reviews that wait for a GitHub reply. Needs `tasks` >= 320 to seed any. */ + externalWaits?: boolean; +} + +export type ReviewAttentionFixture = Awaited>; + +export const REVIEW_ATTENTION_FIXTURE_USER = "review-attention-board"; +export const REVIEW_ATTENTION_FIXTURE_REVIEWER = "review-attention-reviewer"; + +const STATUS_BY_SLOT = [ + "in_review", "in_review", "blocked", "blocked", "cancelled", "backlog", "todo", "todo", "in_progress", "done", + "done", "done", "done", "done", "done", "done", "done", "done", "done", "done", +] as const; + +async function insertChunks(rows: Row[], write: (chunk: Row[]) => Promise) { + for (let start = 0; start < rows.length; start += 500) await write(rows.slice(start, start + 500)); +} + +export async function seedReviewAttentionFixture(db: Db, options: ReviewAttentionFixtureOptions) { + const companyId = randomUUID(); + const now = Date.now(); + const descriptionBytes = options.descriptionBytes ?? 2_000; + await db.insert(companies).values({ + id: companyId, name: `${options.prefix} review attention`, issuePrefix: options.prefix, + requireBoardApprovalForNewAgents: false, + }); + for (const id of [REVIEW_ATTENTION_FIXTURE_USER, REVIEW_ATTENTION_FIXTURE_REVIEWER]) { + await db.insert(authUsers).values({ + id, name: id === REVIEW_ATTENTION_FIXTURE_USER ? "Board Member" : "Reviewer", email: `${id}@example.test`, + createdAt: new Date(now), updatedAt: new Date(now), + }).onConflictDoNothing(); + await db.insert(companyMemberships).values({ + companyId, principalType: "user", principalId: id, status: "active", membershipRole: "owner", + }); + } + + // A reporting tree of five reports per manager. The root and the reviewer + // (index 1) stay invokable; some of the others are paused, terminated or + // waiting for approval, so reviews and blockers meet uninvokable owners. + const agentStatus = (index: number) => index < 2 ? "idle" + : index % 11 === 5 ? "paused" + : index % 13 === 7 ? "terminated" + : index % 17 === 9 ? "pending_approval" + : index % 3 === 0 ? "running" : "idle"; + const agentRows = Array.from({ length: options.agents }, (_unused, index) => ({ + id: randomUUID(), companyId, name: `${options.prefix} agent ${index}`, role: index === 0 ? "ceo" : "engineer", + status: agentStatus(index), adapterType: "process", + })); + const agentIds = agentRows.map(agent => agent.id); + // Managers are inserted before their reports. + await insertChunks(agentRows.map((agent, index) => ({ + ...agent, reportsTo: index === 0 ? null : agentIds[Math.floor((index - 1) / 5)]!, + })), chunk => db.insert(agents).values(chunk)); + const invokableAgentIds = agentRows.filter((_agent, index) => { + for (let current = index; ; current = Math.floor((current - 1) / 5)) { + if (agentStatus(current) === "terminated") return false; + if (current === index && !["idle", "running"].includes(agentStatus(current))) return false; + if (current === 0) return true; + } + }).map(agent => agent.id); + const pausedAgentIds = agentRows.filter((_agent, index) => agentStatus(index) === "paused").map(agent => agent.id); + const pick = (values: Value[], index: number) => values[index % values.length]!; + + type Task = typeof issues.$inferInsert & { id: string; status: string }; + const tasks: Task[] = Array.from({ length: options.tasks }, (_unused, index) => ({ + id: randomUUID(), companyId, issueNumber: index + 1, identifier: `${options.prefix}-${index + 1}`, + title: `${options.prefix} task ${index + 1}`, + description: `Task ${index + 1}. ${"x".repeat(descriptionBytes)}`, + status: STATUS_BY_SLOT[index % STATUS_BY_SLOT.length]!, priority: "medium", + assigneeAgentId: pick(agentIds, index * 7), createdByAgentId: pick(agentIds, index * 3), + createdAt: new Date(now - (options.tasks - index) * 60_000), + updatedAt: new Date(now - (options.tasks - index) * 30_000), + })); + // Four of every five tasks are children of the first task of their group. + for (const [index, task] of tasks.entries()) if (index % 5 !== 0) task.parentId = tasks[index - index % 5]!.id; + const byStatus = (status: string) => tasks.filter(task => task.status === status); + const reviews = byStatus("in_review"), blocked = byStatus("blocked"); + const todo = byStatus("todo"), backlog = byStatus("backlog"), cancelled = byStatus("cancelled"); + const done = byStatus("done"), inProgress = byStatus("in_progress"); + + const reviewsOf = (...variants: number[]) => reviews.filter((_review, index) => variants.includes(index % 16)); + for (const review of reviewsOf(0, 1, 2, 3)) review.assigneeAgentId = pick(invokableAgentIds, tasks.indexOf(review)); + for (const review of reviewsOf(4)) { + review.assigneeAgentId = null; + review.assigneeUserId = REVIEW_ATTENTION_FIXTURE_REVIEWER; + } + // A review stage that waits for its participant. + const waitingFor = (currentParticipant: Record | null) => ({ + status: "pending", currentStageId: randomUUID(), currentStageIndex: 0, currentStageType: "review", currentParticipant, + returnAssignee: null, completedStageIds: [], lastDecisionId: null, lastDecisionOutcome: null, + }); + for (const [index, review] of reviewsOf(5).entries()) { + review.executionState = waitingFor({ type: "agent", agentId: pick(invokableAgentIds, index), userId: null }); + } + for (const [index, review] of reviewsOf(6).entries()) { + review.executionState = waitingFor({ type: "agent", agentId: pick(pausedAgentIds, index), userId: null }); + } + for (const review of reviewsOf(7)) { + review.executionState = waitingFor({ type: "user", userId: REVIEW_ATTENTION_FIXTURE_REVIEWER, agentId: null }); + } + for (const review of reviewsOf(8)) { + review.monitorNextCheckAt = new Date(now + 3_600_000); + review.executionPolicy = { monitor: { maxAttempts: 3 } }; + } + for (const review of reviewsOf(14)) review.executionState = waitingFor(null); + const externalWaits = options.externalWaits === false ? [] : reviewsOf(15); + // The endpoint that waits for the reply is assigned to the reviewer agent. + for (const review of reviewsOf(15)) review.assigneeAgentId = agentIds[1]!; + + for (const [index, task] of blocked.entries()) { + task.blockedTransitionAt = new Date(now - 3_600_000); + if (index % 10 === 9) { + task.unblockDescriptor = { owner: "board", action: "Choose how to continue this task." } as never; + } + } + for (const [index, task] of todo.entries()) { + if (index % 3 === 0) task.assigneeAgentId = null; + if (index % 3 === 1) task.assigneeAgentId = pick(pausedAgentIds, index); + } + await insertChunks(tasks, chunk => db.insert(issues).values(chunk)); + + const stalledReviews = reviewsOf(0, 1, 2, 3); + const relations: Array<{ blocker: Task; blocked: Task }> = []; + for (const [index, task] of blocked.entries()) { + const variant = index % 10; + const next = blocked[index + 1]; + if (variant === 0) relations.push({ blocker: pick(stalledReviews, index), blocked: task }); + if (variant === 1) relations.push({ blocker: pick(todo.filter((_task, at) => at % 3 === 0), index), blocked: task }); + if (variant === 2) relations.push({ blocker: pick(backlog, index), blocked: task }); + if (variant === 3) relations.push({ blocker: pick(cancelled, index), blocked: task }); + if (variant === 4) relations.push({ blocker: pick(done, index), blocked: task }); + // A chain: this task waits for the next blocked task, which waits for a + // task whose assignee is paused. + if (variant === 5 && next) relations.push({ blocker: next, blocked: task }); + if (variant === 6) relations.push({ blocker: pick(todo.filter((_task, at) => at % 3 === 1), index), blocked: task }); + // A cycle of two blocked tasks. The second also waits for running work. + if (variant === 7 && next) relations.push({ blocker: next, blocked: task }, { blocker: task, blocked: next }); + if (variant === 8) relations.push({ blocker: pick(inProgress, index), blocked: task }); + if (variant === 9) relations.push({ blocker: pick(inProgress, index + 1), blocked: task }); + } + await insertChunks(relations, chunk => db.insert(issueRelations).values(chunk.map(relation => ({ + companyId, issueId: relation.blocker.id, relatedIssueId: relation.blocked.id, type: "blocks" as const, + })))); + + await insertChunks(reviewsOf(9), chunk => db.insert(issueThreadInteractions).values(chunk.map((review, index) => ({ + companyId, issueId: review.id, kind: "request_confirmation", status: "pending", continuationPolicy: "wake_assignee", + ...(index % 2 === 0 ? {} : { + requestedResolverPolicy: "human_only" as const, effectiveResolverPolicy: "human_only" as const, + resolverPolicyProvenance: "explicit" as const, effectiveResolverPolicySource: "requested" as const, + }), + payload: { version: 1, prompt: "Accept this work?" } as never, + })))); + const approvalRows = reviewsOf(10).map(review => ({ id: randomUUID(), issueId: review.id })); + await insertChunks(approvalRows, chunk => db.insert(approvals).values(chunk.map(row => ({ + id: row.id, companyId, type: "request_board_approval", status: "pending", payload: { title: "Review" }, + })))); + await insertChunks(approvalRows, chunk => db.insert(issueApprovals).values(chunk.map(row => ({ + companyId, issueId: row.issueId, approvalId: row.id, + })))); + await insertChunks(reviewsOf(11), chunk => db.insert(agentWakeupRequests).values(chunk.map(review => ({ + companyId, agentId: review.assigneeAgentId!, source: "automation", reason: "issue_commented", status: "queued", + payload: { issueId: review.id }, + })))); + // Running work on a review, and on the tasks that block other tasks. + await insertChunks([...reviewsOf(12), ...inProgress], chunk => db.insert(heartbeatRuns).values(chunk.map(task => ({ + companyId, agentId: task.assigneeAgentId!, status: "running", contextSnapshot: { issueId: task.id }, + })))); + await insertChunks(reviewsOf(13), chunk => db.insert(issueRecoveryActions).values(chunk.map(review => ({ + companyId, sourceIssueId: review.id, kind: "missing_disposition", status: "active", ownerType: "agent", + ownerAgentId: review.assigneeAgentId!, cause: "review_path_lost", fingerprint: `review-path:${review.id}`, + evidence: {}, nextAction: "Restore the review path", + })))); + + // Open recovery tasks. Their origin names a stalled review (which they then + // cover), a blocked task, or a task that no longer waits. + const recoveryTargets = [...stalledReviews.filter((_review, index) => index % 4 === 3), ...blocked, ...done]; + const recoveryTasks = Array.from({ length: options.recoveryTasks ?? Math.floor(options.tasks / 20) }, (_unused, index) => { + const target = pick(recoveryTargets, index); + const stranded = index % 2 === 0; + return { + id: randomUUID(), companyId, issueNumber: options.tasks + index + 1, + identifier: `${options.prefix}-${options.tasks + index + 1}`, title: `Recover ${target.identifier}`, + status: "todo", priority: "high", assigneeAgentId: pick(invokableAgentIds, index), + originKind: stranded ? RECOVERY_ORIGIN_KINDS.strandedIssueRecovery : RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + originId: stranded ? target.id : buildIssueGraphLivenessIncidentKey({ + companyId, issueId: target.id, state: "in_review_without_action_path", blockerIssueId: target.id, + }), + originFingerprint: `recovery:${index}`, + }; + }); + await insertChunks(recoveryTasks, chunk => db.insert(issues).values(chunk)); + + if (externalWaits.length > 0) await seedExternalReviewWaits(db, companyId, agentIds[1]!, externalWaits.map(review => review.id)); + + // What the fixture is built to produce, stated without the classification: + // the kinds of the paths that cover each review, or why it is stalled. + const kindsByVariant: Record = { + 4: ["human_reviewer"], 5: ["execution_participant"], 7: ["execution_participant"], 8: ["monitor"], + 9: ["interaction"], 10: ["approval"], 11: ["queued_wake"], 12: ["active_run"], 13: ["recovery"], + 15: externalWaits.length > 0 ? ["external_response"] : [], + }; + const recoveredIds = recoveryTasks.map((_task, index) => pick(recoveryTargets, index).id); + const expectedReviews = reviews.map((review, index) => { + const variant = index % 16; + const kinds = [...(kindsByVariant[variant] ?? []), ...recoveredIds.filter(id => id === review.id).map(() => "recovery")]; + return { + id: review.id, identifier: review.identifier!, variant, kinds, + stalledReason: kinds.length > 0 ? null + : variant === 6 ? "current participant agent is paused" + : variant === 14 ? "its current participant cannot be resolved" + : "is in review with an agent assignee but no participant", + }; + }); + + return { + companyId, userId: REVIEW_ATTENTION_FIXTURE_USER, agentIds, taskIds: tasks.map(task => task.id), + reviewIds: reviews.map(review => review.id), blockedIds: blocked.map(task => task.id), + externalWaitIds: externalWaits.map(review => review.id), expectedReviews, + counts: { + tasks: tasks.length, agents: agentRows.length, reviews: reviews.length, blocked: blocked.length, + relations: relations.length, recoveryTasks: recoveryTasks.length, externalWaits: externalWaits.length, + }, + }; +} + +/** + * Each task waits in review for the next GitHub reply to a published review: + * its last status decision moved it there, and the run that made the decision + * succeeded on a conversation of an endpoint assigned to the task's agent. + */ +async function seedExternalReviewWaits(db: Db, companyId: string, agentId: string, issueIds: string[]) { + const sql = db.$client; + const applicationId = randomUUID(), connectionId = randomUUID(), endpointId = randomUUID(), resourceId = randomUUID(); + const principalId = randomUUID(), user = REVIEW_ATTENTION_FIXTURE_REVIEWER; + const repository = "synthetic/repository", repositoryId = "42", headSha = "1".repeat(40); + await db.insert(toolApplications).values({ id: applicationId, companyId, name: "GitHub", type: "chat" }); + await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "GitHub", + uid: connectionId, transport: "chat_sdk", connectionPurpose: "channel", enabled: true, status: "active" }); + await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, provider: "github", + publicId: endpointId, assignedAgentId: agentId, status: "active", providerAccountId: `synthetic-${companyId}` }); + await db.insert(chatEndpointResources).values({ id: resourceId, companyId, endpointId, type: "repository", + providerResourceId: repository, label: repository, metadata: { providerRepositoryId: repositoryId }, + availability: "available", enabled: true }); + await db.insert(chatExternalPrincipals).values({ id: principalId, companyId, provider: "github", + providerAccountId: `synthetic-${companyId}`, externalId: "1", kind: "user" }); + await db.insert(chatIdentityLinks).values({ companyId, endpointId, principalId, paperclipUserId: user, status: "linked" }); + + const waits = issueIds.map((issueId, index) => ({ + issueId, pullNumber: index + 1, runId: randomUUID(), reviewRunId: randomUUID(), reviewId: randomUUID(), + decisionId: randomUUID(), conversationId: randomUUID(), commentId: randomUUID(), deliveryId: randomUUID(), + contextId: randomUUID(), + })); + await sql.begin(async tx => { + // Skip the work-assessment owner chain of the status decisions. + await tx.unsafe("set local session_replication_role = replica"); + await tx.unsafe(`update issues set last_status_decision_id = w.decision_id + from unnest($1::uuid[], $2::uuid[]) as w(issue_id, decision_id) where issues.id = w.issue_id`, + [waits.map(wait => wait.issueId), waits.map(wait => wait.decisionId)]); + await tx.unsafe(`insert into status_decisions (id, company_id, issue_id, run_id, assessment_id, decision_version, policy_version, + from_status, to_status, reason_code, decision_json, decision_digest, application_state, applied_at) + select w.decision_id, $1::uuid, w.issue_id, w.run_id, gen_random_uuid(), 1, 'v1', 'in_progress', 'in_review', + 'external_chat_response_waiting', '{}'::jsonb, w.decision_id::text, 'applied', now() + from unnest($2::uuid[], $3::uuid[], $4::uuid[]) as w(decision_id, issue_id, run_id)`, + [companyId, waits.map(wait => wait.decisionId), waits.map(wait => wait.issueId), waits.map(wait => wait.runId)]); + }); + await insertChunks(waits, chunk => db.insert(heartbeatRuns).values(chunk.flatMap(wait => [ + { id: wait.runId, companyId, agentId, status: "succeeded", runtimeMode: "native", nativeIssueId: wait.issueId, + responsibleUserId: user, activeIdentityContextId: wait.contextId, contextSnapshot: { prompt: "x".repeat(1024) } }, + { id: wait.reviewRunId, companyId, agentId, status: "succeeded", nativeIssueId: wait.issueId }, + ]))); + await insertChunks(waits, chunk => db.insert(issueComments).values(chunk.map(wait => ({ id: wait.commentId, companyId, + issueId: wait.issueId, authorType: "user" as const, authorUserId: user, body: "Review" })))); + await insertChunks(waits, chunk => db.insert(runIdentityContexts).values(chunk.map(wait => ({ id: wait.contextId, + companyId, runId: wait.runId, revision: 1, responsibleUserId: user, messageId: wait.commentId, parentContextId: null, + cause: "instruction", correlationId: "fixture", status: "accepted" })))); + await insertChunks(waits, chunk => db.insert(chatConversations).values(chunk.map(wait => ({ id: wait.conversationId, + companyId, endpointId, resourceId, issueId: wait.issueId, externalConversationId: repository, + externalThreadId: `github:${repository}:${wait.pullNumber}`, externalLabel: "Review", state: "waiting" })))); + await insertChunks(waits, chunk => db.insert(chatDeliveries).values(chunk.map(wait => ({ id: wait.deliveryId, companyId, + endpointId, conversationId: wait.conversationId, principalId, providerEventId: wait.deliveryId, + deduplicationKey: wait.deliveryId, eventKind: "message" as const, state: "processed" as const, normalizedEvent: {}, + processedAt: new Date() })))); + await insertChunks(waits, chunk => db.insert(chatMessageLinks).values(chunk.map(wait => ({ companyId, endpointId, + conversationId: wait.conversationId, deliveryId: wait.deliveryId, commentId: wait.commentId, + providerMessageId: wait.commentId, direction: "inbound" })))); + await insertChunks(waits, chunk => db.insert(chatGitHubReviews).values(chunk.map(wait => ({ id: wait.reviewId, companyId, + endpointId, issueId: wait.issueId, runId: wait.reviewRunId, repositoryId, repository, pullNumber: wait.pullNumber, + headSha, deliveryId: wait.reviewId, configurationRevision: 1, state: "completed" as const, + policySnapshot: defaultGitHubReviewPolicy(), + event: { event: "mention" as const, deliveryId: wait.reviewId, repositoryId, repository, pullNumber: wait.pullNumber, + title: "Review", body: "", baseSha: "0".repeat(40), headSha, baseBranch: "main", + author: { id: "1", login: "author", isBot: false }, sender: { id: "1", login: "author" }, draft: false, labels: [] }, + assessment: { reviewedCommit: headSha, score: 5 as const, complete: true, summary: "Reviewed", rationale: "Reviewed", + coverage: { reviewedPaths: [], omittedPaths: [], limitations: [] }, findings: [] } })))); + await insertChunks(waits, chunk => db.insert(chatActions).values(chunk.map(wait => ({ companyId, endpointId, + kind: "github_review_publication", status: "processed", + providerActionId: `github_publication:${wait.issueId}:assessment:${createHash("sha256").update(JSON.stringify(wait.reviewId)).digest("hex")}` })))); +} diff --git a/server/src/__tests__/issue-graph-liveness-equivalence.test.ts b/server/src/__tests__/issue-graph-liveness-equivalence.test.ts new file mode 100644 index 00000000000..c304761b506 --- /dev/null +++ b/server/src/__tests__/issue-graph-liveness-equivalence.test.ts @@ -0,0 +1,272 @@ +import { describe, expect, it } from "vitest"; +import { + classifyIssueGraphLiveness, + classifyIssueReviewPaths, + createIssueReviewPathClassifier, + type IssueGraphLivenessInput, + type IssueLivenessAgentInput, + type IssueLivenessExecutionPathInput, + type IssueLivenessIssueInput, + type IssueLivenessWaitingPathInput, +} from "../services/recovery/issue-graph-liveness.js"; +import { + referenceClassifyIssueGraphLiveness, + referenceClassifyIssueReviewPaths, +} from "./helpers/issue-graph-liveness-reference.js"; + +const NOW = new Date("2026-11-12T12:00:00.000Z"); +const COMPANIES = ["company-a", "company-b"]; + +/** A deterministic pseudo-random source, so a failing case can be replayed by its seed. */ +function randomSource(seed: number) { + let state = seed >>> 0; + const next = () => { + state = (state + 0x6d2b79f5) >>> 0; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296; + }; + const int = (max: number) => Math.floor(next() * max); + return { + int, + chance: (probability: number) => next() < probability, + pick: (values: readonly Value[]) => values[int(values.length)]!, + }; +} + +/** + * A task graph of two companies that reaches the rules of the classification: + * reviews with every kind of path and without one, blocked chains and cycles, + * terminal blockers, relations and paths that name another company or a task + * that does not exist, reporting chains with cycles, missing managers and + * managers of another company, and ids that the input lists twice. + */ +function randomInput(seed: number): IssueGraphLivenessInput { + const random = randomSource(seed); + const agentCount = 2 + random.int(10), issueCount = 4 + random.int(22); + const agentId = (index: number) => `agent-${index}`; + const issueId = (index: number) => `issue-${index}`; + const anyAgentId = () => random.chance(0.05) ? "agent-missing" : agentId(random.int(agentCount)); + const anyIssueId = () => random.chance(0.05) ? "issue-missing" : issueId(random.int(issueCount)); + const company = () => random.chance(0.8) ? COMPANIES[0]! : COMPANIES[1]!; + + const agents: IssueLivenessAgentInput[] = Array.from({ length: agentCount }, (_unused, index) => ({ + id: agentId(index), companyId: company(), name: `Agent ${index}`, role: "engineer", title: null, + status: random.pick(["idle", "idle", "running", "active", "error", "paused", "terminated", "pending_approval", "unknown"]), + reportsTo: random.chance(0.3) ? null : anyAgentId(), + })); + if (random.chance(0.2)) agents.push({ ...random.pick(agents), status: "paused", reportsTo: null }); + + const participant = () => random.pick([ + null, undefined, {}, "agent", { type: "agent", agentId: anyAgentId() }, { type: "agent", agentId: "" }, + { type: "user", userId: "user-1" }, { type: "user", userId: "" }, { type: "user" }, + ]); + const issues: IssueLivenessIssueInput[] = Array.from({ length: issueCount }, (_unused, index) => { + const conversation = random.chance(0.1); + return { + id: issueId(index), companyId: company(), identifier: random.chance(0.9) ? `T-${index}` : null, title: `Task ${index}`, + status: random.pick(["in_review", "in_review", "in_review", "blocked", "blocked", "todo", "backlog", "in_progress", "done", "cancelled"]), + assigneeAgentId: random.chance(0.75) ? anyAgentId() : null, + assigneeUserId: random.chance(0.12) ? "user-1" : null, + createdByAgentId: random.chance(0.5) ? anyAgentId() : null, + createdByUserId: null, + conversationAgentId: conversation ? anyAgentId() : null, + conversationUserId: conversation ? "user-2" : null, + conversationState: conversation ? random.pick(["waiting", "active"]) : null, + executionState: random.pick | null>([ + null, null, null, { status: "pending", currentParticipant: participant() }, + { status: "completed", currentParticipant: participant() }, { monitor: { maxAttempts: 2, attemptCount: random.int(4) } }, + ]), + executionPolicy: random.pick | null>([ + null, null, { monitor: { maxAttempts: 1 + random.int(3) } }, + { monitor: { timeoutAt: new Date(NOW.getTime() + (random.int(3) - 1) * 60_000).toISOString() } }, + ]), + monitorNextCheckAt: random.pick([null, null, new Date(NOW.getTime() + 60_000), new Date(NOW.getTime() - 60_000), "not a date"]), + monitorAttemptCount: random.pick([null, 0, 1, 5]), + }; + }); + if (random.chance(0.15)) issues.push({ ...random.pick(issues), status: "in_review" }); + + const relations = Array.from({ length: random.int(issueCount * 2) }, () => ({ + companyId: company(), blockerIssueId: anyIssueId(), blockedIssueId: anyIssueId(), + })); + const executionPaths = (): IssueLivenessExecutionPathInput[] => Array.from({ length: random.int(8) }, (_unused, index) => ({ + id: random.chance(0.9) ? `execution-${index}` : null, companyId: company(), + issueId: random.chance(0.1) ? null : anyIssueId(), agentId: random.chance(0.8) ? anyAgentId() : null, + status: "queued", createdAt: random.chance(0.7) ? new Date(NOW.getTime() - index * 1_000) : null, + })); + const waitingPaths = (): IssueLivenessWaitingPathInput[] => Array.from({ length: random.int(8) }, (_unused, index) => ({ + id: random.chance(0.9) ? `waiting-${index}` : null, companyId: company(), issueId: anyIssueId(), + status: "pending", createdAt: random.chance(0.7) ? new Date(NOW.getTime() - index * 1_000).toISOString() : null, + })); + const optional = (build: () => Value) => random.chance(0.85) ? build() : undefined; + return { + issues, relations, agents, now: NOW, + activeRuns: optional(executionPaths), queuedWakeRequests: optional(executionPaths), + pendingInteractions: optional(waitingPaths), pendingApprovals: optional(waitingPaths), + externalResponseWaits: optional(waitingPaths), openRecoveryIssues: optional(waitingPaths), + }; +} + +function agent(index: number, reportsTo: string | null, status = "idle"): IssueLivenessAgentInput { + return { id: `agent-${index}`, companyId: COMPANIES[0]!, name: `Agent ${index}`, role: "engineer", title: null, status, reportsTo }; +} + +function review(index: number, overrides: Partial = {}): IssueLivenessIssueInput { + return { + id: `review-${index}`, companyId: COMPANIES[0]!, identifier: `R-${index}`, title: `Review ${index}`, status: "in_review", + assigneeAgentId: "agent-1", assigneeUserId: null, createdByAgentId: null, createdByUserId: null, executionState: null, + ...overrides, + }; +} + +/** Counts how often a classification reads the `status` of an agent or the `issueId` of a path. */ +function countedInput(reviews: number) { + const reads = { agentStatus: 0, pathIssueId: 0 }; + const counted = (row: Row, key: keyof Row & string, counter: keyof typeof reads) => { + const value = row[key]; + return Object.defineProperty({ ...row }, key, { enumerable: true, get: () => { reads[counter] += 1; return value; } }) as Row; + }; + // Forty agents under one manager; every third is paused. + const agents = Array.from({ length: 40 }, (_unused, index) => + counted(agent(index, index === 0 ? null : "agent-0", index % 3 === 2 ? "paused" : "idle"), "status", "agentStatus")); + const waitingPaths = (prefix: string) => Array.from({ length: 60 }, (_unused, index) => counted({ + id: `${prefix}-${index}`, companyId: COMPANIES[0]!, issueId: `elsewhere-${index}`, status: "pending", createdAt: NOW, + }, "issueId", "pathIssueId")); + const executionPaths = (prefix: string) => Array.from({ length: 60 }, (_unused, index) => counted({ + id: `${prefix}-${index}`, companyId: COMPANIES[0]!, issueId: `elsewhere-${index}` as string | null, agentId: "agent-1", + status: "queued", createdAt: NOW, + }, "issueId", "pathIssueId")); + const input: IssueGraphLivenessInput = { + // Every review is stalled: an invokable assignee and no path. + issues: Array.from({ length: reviews }, (_unused, index) => review(index)), + relations: [], agents, now: NOW, + activeRuns: executionPaths("run"), queuedWakeRequests: executionPaths("wake"), + pendingInteractions: waitingPaths("interaction"), pendingApprovals: waitingPaths("approval"), + externalResponseWaits: waitingPaths("external"), openRecoveryIssues: waitingPaths("recovery"), + }; + return { input, reads }; +} + +describe("issue graph liveness: one index for every task of a classification", () => { + it("returns the findings and review paths of the pre-fix classification for random task graphs", () => { + const findingStates = new Map(), pathKinds = new Map(); + const count = (counts: Map, key: string) => counts.set(key, (counts.get(key) ?? 0) + 1); + for (let seed = 1; seed <= 400; seed += 1) { + const input = randomInput(seed); + const expected = referenceClassifyIssueGraphLiveness(input); + expect(classifyIssueGraphLiveness(input), `findings of seed ${seed}`).toEqual(expected); + for (const finding of expected) count(findingStates, finding.state); + + const reviewPathsOf = createIssueReviewPathClassifier(input); + for (const issue of input.issues) { + const expectedPaths = referenceClassifyIssueReviewPaths(input, issue); + expect(classifyIssueReviewPaths(input, issue), `review paths of ${issue.id}, seed ${seed}`).toEqual(expectedPaths); + expect(reviewPathsOf(issue), `indexed review paths of ${issue.id}, seed ${seed}`).toEqual(expectedPaths); + for (const path of expectedPaths) count(pathKinds, path.kind); + } + } + // The random graphs reach every finding and every kind of review path, + // each of them many times. + expect([...findingStates.keys()].sort()).toEqual([ + "blocked_by_assigned_backlog_issue", "blocked_by_cancelled_issue", "blocked_by_unassigned_issue", + "blocked_by_uninvokable_assignee", "in_review_without_action_path", "invalid_review_participant", + ]); + expect([...pathKinds.keys()].sort()).toEqual([ + "active_run", "approval", "execution_participant", "external_response", "human_reviewer", "interaction", + "monitor", "queued_wake", "recovery", + ]); + expect(Math.min(...findingStates.values(), ...pathKinds.values())).toBeGreaterThanOrEqual(20); + }); + + it("returns the pre-fix findings for blocked chains, cycles and terminal blockers", () => { + const agents = [agent(0, null), agent(1, "agent-0"), agent(2, "agent-0", "paused"), agent(3, "agent-2", "terminated")]; + const task = (id: string, status: string, overrides: Partial = {}): IssueLivenessIssueInput => ({ + id, companyId: COMPANIES[0]!, identifier: id.toUpperCase(), title: id, status, assigneeAgentId: "agent-1", + assigneeUserId: null, createdByAgentId: "agent-1", createdByUserId: null, executionState: null, ...overrides, + }); + const blocks = (blockerIssueId: string, blockedIssueId: string, companyId = COMPANIES[0]!) => ({ companyId, blockerIssueId, blockedIssueId }); + const input: IssueGraphLivenessInput = { + now: NOW, + agents, + issues: [ + task("chain-1", "blocked"), task("chain-2", "blocked"), task("chain-3", "blocked"), + task("chain-leaf", "todo", { assigneeAgentId: null }), + task("cycle-1", "blocked"), task("cycle-2", "blocked"), + task("behind-cancelled", "blocked"), task("cancelled", "cancelled"), + task("behind-done", "blocked"), task("done", "done"), + task("behind-backlog", "blocked"), task("backlog", "backlog"), + task("behind-paused", "blocked"), task("paused-owner", "todo", { assigneeAgentId: "agent-2" }), + task("behind-terminated-chain", "blocked"), task("terminated-chain-owner", "todo", { assigneeAgentId: "agent-3" }), + task("behind-review", "blocked"), task("stalled-review", "in_review"), + task("behind-covered-review", "blocked"), task("covered-review", "in_review"), + task("behind-running", "blocked"), task("running", "in_progress"), + task("behind-other-company", "blocked"), { ...task("other-company", "todo", { assigneeAgentId: null }), companyId: COMPANIES[1]! }, + task("waiting-external", "in_review"), + task("invalid-participant", "in_review", { executionState: { status: "pending", currentParticipant: { type: "agent", agentId: "agent-2" } } }), + ], + relations: [ + blocks("chain-2", "chain-1"), blocks("chain-3", "chain-2"), blocks("chain-leaf", "chain-3"), + blocks("cycle-2", "cycle-1"), blocks("cycle-1", "cycle-2"), + blocks("cancelled", "behind-cancelled"), blocks("done", "behind-done"), blocks("backlog", "behind-backlog"), + blocks("paused-owner", "behind-paused"), blocks("terminated-chain-owner", "behind-terminated-chain"), + blocks("stalled-review", "behind-review"), blocks("covered-review", "behind-covered-review"), + blocks("running", "behind-running"), blocks("other-company", "behind-other-company"), + blocks("chain-leaf", "behind-other-company", COMPANIES[1]!), + ], + activeRuns: [{ id: "run-1", companyId: COMPANIES[0]!, issueId: "running", agentId: "agent-1", status: "running", createdAt: NOW }], + queuedWakeRequests: [{ id: "wake-other", companyId: COMPANIES[1]!, issueId: "stalled-review", agentId: "agent-1", status: "queued" }], + pendingInteractions: [{ id: "interaction-1", companyId: COMPANIES[0]!, issueId: "covered-review", status: "pending", createdAt: NOW }], + externalResponseWaits: [{ id: "wait-1", companyId: COMPANIES[0]!, issueId: "waiting-external", status: "waiting", createdAt: NOW }], + }; + + const findings = classifyIssueGraphLiveness(input); + expect(findings).toEqual(referenceClassifyIssueGraphLiveness(input)); + expect(Object.fromEntries(findings.map((finding) => [finding.issueId, finding.state]))).toEqual({ + "chain-1": "blocked_by_unassigned_issue", + "behind-cancelled": "blocked_by_cancelled_issue", + "behind-backlog": "blocked_by_assigned_backlog_issue", + "behind-paused": "blocked_by_uninvokable_assignee", + "behind-terminated-chain": "blocked_by_uninvokable_assignee", + "behind-review": "in_review_without_action_path", + "invalid-participant": "invalid_review_participant", + }); + // A wake of another company does not cover the review; the external wait does. + const reviewPathsOf = createIssueReviewPathClassifier(input); + const kinds = (id: string) => reviewPathsOf(input.issues.find((issue) => issue.id === id)!).map((path) => path.kind); + expect(kinds("stalled-review")).toEqual([]); + expect(kinds("covered-review")).toEqual(["interaction"]); + expect(kinds("waiting-external")).toEqual(["external_response"]); + }); + + it("reads each agent and each path a number of times that does not grow with the reviews", () => { + const few = countedInput(20), many = countedInput(400); + const fewFindings = classifyIssueGraphLiveness(few.input), manyFindings = classifyIssueGraphLiveness(many.input); + expect(fewFindings).toHaveLength(20); + expect(manyFindings).toHaveLength(400); + expect(manyFindings[0]!.recommendedOwnerCandidateAgentIds).toHaveLength(27); + // Twenty times the reviews, the same reads: each of the 360 paths once + // and each of the 40 agents a few times. + expect(many.reads).toEqual(few.reads); + expect(many.reads.pathIssueId).toBe(360); + expect(many.reads.agentStatus).toBeLessThanOrEqual(40 * 10); + + // The pre-fix classification is what this test catches: it read every + // agent and every path again for each review. + const referenceFew = countedInput(20), referenceMany = countedInput(400); + expect(referenceClassifyIssueGraphLiveness(referenceMany.input)).toEqual(manyFindings); + referenceClassifyIssueGraphLiveness(referenceFew.input); + expect(referenceMany.reads.agentStatus).toBeGreaterThan(referenceFew.reads.agentStatus * 15); + expect(referenceMany.reads.pathIssueId).toBeGreaterThan(referenceFew.reads.pathIssueId * 15); + }); + + it("classifies the review paths of a page from one index", () => { + const few = countedInput(20), many = countedInput(400); + for (const { input } of [few, many]) { + const reviewPathsOf = createIssueReviewPathClassifier(input); + for (const issue of input.issues) expect(reviewPathsOf(issue)).toEqual([]); + } + expect(many.reads).toEqual(few.reads); + }); +}); diff --git a/server/src/__tests__/review-attention-classification-equivalence.test.ts b/server/src/__tests__/review-attention-classification-equivalence.test.ts new file mode 100644 index 00000000000..305e88d30dd --- /dev/null +++ b/server/src/__tests__/review-attention-classification-equivalence.test.ts @@ -0,0 +1,208 @@ +import express from "express"; +import request from "supertest"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { createDb, inboxDismissals } from "@paperclipai/db"; +import type { AttentionFeed, AttentionSummary, Issue } from "@paperclipai/shared"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; +import { seedReviewAttentionFixture, type ReviewAttentionFixture } from "./helpers/review-attention-fixture.js"; + +// Every classification the routes run is also run by the pre-fix +// implementation on the same input. A difference fails the test. +const shadow = vi.hoisted(() => ({ + mismatches: [] as string[], + classifications: 0, + graphsWithRelations: 0, + findingStates: new Set(), + pathKinds: new Set(), + reviewsClassified: 0, +})); + +vi.mock("../services/recovery/issue-graph-liveness.js", async (importOriginal) => { + const actual = await importOriginal(); + const reference = await import("./helpers/issue-graph-liveness-reference.js"); + const same = (label: string, current: unknown, expected: unknown) => { + if (JSON.stringify(current) !== JSON.stringify(expected)) shadow.mismatches.push(label); + }; + return { + ...actual, + classifyIssueGraphLiveness(input: Parameters[0]) { + const findings = actual.classifyIssueGraphLiveness(input); + same(`findings of ${input.issues.length} tasks`, findings, reference.referenceClassifyIssueGraphLiveness(input)); + shadow.classifications += 1; + if (input.relations.length > 0) shadow.graphsWithRelations += 1; + for (const finding of findings) shadow.findingStates.add(finding.state); + return findings; + }, + createIssueReviewPathClassifier(input: Parameters[0]) { + const reviewPathsOf = actual.createIssueReviewPathClassifier(input); + return (issue: Parameters[0]) => { + const paths = reviewPathsOf(issue); + same(`review paths of ${issue.identifier}`, paths, reference.referenceClassifyIssueReviewPaths(input, issue)); + shadow.reviewsClassified += 1; + for (const path of paths) shadow.pathKinds.add(path.kind); + return paths; + }; + }, + }; +}); + +const { errorHandler } = await import("../middleware/index.js"); +const { attentionRoutes } = await import("../routes/attention.js"); +const { issueRoutes } = await import("../routes/issues.js"); + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping embedded Postgres review attention equivalence tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +type ListedIssue = Issue & { blockedInboxAttention?: { state: string } }; + +describeEmbeddedPostgres("review attention through the routes: indexed classification", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + let fixture!: ReviewAttentionFixture; + let other!: ReviewAttentionFixture; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-review-attention-equivalence-"); + db = createDb(tempDb.connectionString); + // Two companies of the same shapes. Each request reads one of them. + fixture = await seedReviewAttentionFixture(db, { tasks: 640, agents: 24, prefix: "RAE" }); + other = await seedReviewAttentionFixture(db, { tasks: 320, agents: 12, prefix: "OTH" }); + }, 120_000); + + beforeEach(() => { + shadow.mismatches.length = 0; + shadow.classifications = 0; + shadow.graphsWithRelations = 0; + shadow.reviewsClassified = 0; + shadow.findingStates.clear(); + shadow.pathKinds.clear(); + }); + + afterAll(async () => { + await db?.$client.end({ timeout: 1 }); + await tempDb?.cleanup(); + }); + + function app(target: ReviewAttentionFixture) { + const testApp = express(); + testApp.use((req, _res, next) => { + (req as unknown as { actor: unknown }).actor = { + type: "board", source: "session", userId: target.userId, companyIds: [target.companyId], isInstanceAdmin: false, + }; + next(); + }); + testApp.use("/api", attentionRoutes(db)); + testApp.use("/api", issueRoutes(db, {} as never)); + testApp.use(errorHandler); + return testApp; + } + + async function read(target: ReviewAttentionFixture, route: string) { + const response = await request(app(target)).get(`/api/companies/${target.companyId}${route}`); + expect(response.status, `${route}: ${JSON.stringify(response.body).slice(0, 300)}`).toBe(200); + return response.body as Body; + } + + const stable = (summary: AttentionSummary) => ({ ...summary, generatedAt: null }); + + it("gives every review the paths the fixture was built with, and the pre-fix classification of every request", async () => { + const reviews = await read(fixture, "/issues?status=in_review&limit=1000"); + expect(reviews).toHaveLength(fixture.expectedReviews.length); + const byId = new Map(reviews.map((row) => [row.id, row.reviewAttention!])); + for (const expected of fixture.expectedReviews) { + const attention = byId.get(expected.id)!; + const label = `${expected.identifier} (variant ${expected.variant})`; + expect(attention.paths.map((path) => path.kind).sort(), label).toEqual([...expected.kinds].sort()); + expect(attention.state, label).toBe(expected.kinds.length > 0 ? "covered" : "stalled"); + if (expected.stalledReason) expect(attention.reason, label).toContain(expected.stalledReason); + } + // The fixture reaches every kind of path, a review that recovery tasks + // cover, and the three ways a review stalls. + expect(new Set(fixture.expectedReviews.flatMap((review) => review.kinds))).toEqual(new Set([ + "human_reviewer", "execution_participant", "monitor", "interaction", "approval", "queued_wake", "active_run", + "recovery", "external_response", + ])); + expect(fixture.expectedReviews.filter((review) => review.variant < 4 && review.kinds.includes("recovery")).length).toBeGreaterThan(0); + expect(new Set(fixture.expectedReviews.map((review) => review.stalledReason)).size).toBe(4); + + // The dashboard task list and the blocked inbox, whose task graph has the + // blocked chains, the cycles and the terminal blockers. + const taskList = await read(fixture, "/issues"); + for (const row of taskList.filter((task) => task.status === "in_review")) { + expect(row.reviewAttention, row.identifier ?? row.id).toEqual(byId.get(row.id)); + } + const blockedInbox = await read(fixture, "/issues?attention=blocked"); + expect(blockedInbox.length).toBeGreaterThan(0); + + expect(shadow.mismatches).toEqual([]); + expect(shadow.reviewsClassified).toBeGreaterThanOrEqual(fixture.expectedReviews.length); + expect(shadow.graphsWithRelations).toBeGreaterThan(0); + expect([...shadow.pathKinds].sort()).toEqual([ + "active_run", "approval", "execution_participant", "external_response", "human_reviewer", "interaction", + "monitor", "queued_wake", "recovery", + ]); + expect([...shadow.findingStates].sort()).toEqual(expect.arrayContaining([ + "blocked_by_assigned_backlog_issue", "blocked_by_cancelled_issue", "blocked_by_unassigned_issue", + "blocked_by_uninvokable_assignee", "in_review_without_action_path", "invalid_review_participant", + ])); + }, 60_000); + + it("counts the reviews that need a person, leaves out the ones a person dismissed, and reads one company", async () => { + const summary = await read(fixture, "/attention/summary"); + const feed = await read(fixture, "/attention?all=true"); + // A review needs a person when it is stalled, assigned to a user, waiting + // for a user participant, or waiting for a pending approval. + const needsPerson = fixture.expectedReviews.filter((review) => + review.kinds.length === 0 || [4, 7, 10].includes(review.variant)); + const stalled = fixture.expectedReviews.filter((review) => review.kinds.length === 0); + expect(summary.countsBySourceKind.review).toBe(needsPerson.length); + expect(summary.totalCount).toBe(feed.items.length); + expect(feed.countsBySourceKind).toEqual(summary.countsBySourceKind); + const reviewCards = feed.items.filter((item) => item.sourceKind === "review"); + const label = (id: string) => { + const expected = fixture.expectedReviews.find((review) => review.id === id); + return expected ? `${expected.identifier} (variant ${expected.variant})` : `task ${fixture.taskIds.indexOf(id) + 1}`; + }; + expect(reviewCards.map((item) => label(item.subject.id)).sort()).toEqual(needsPerson.map((review) => label(review.id)).sort()); + expect(reviewCards.filter((item) => item.inlineResolvable).map((item) => label(item.subject.id)).sort()) + .toEqual(stalled.map((review) => label(review.id)).sort()); + + // The other company has its own counts, and its requests read none of + // this company's tasks. + const otherSummary = await read(other, "/attention/summary"); + const otherFeed = await read(other, "/attention?all=true"); + expect(otherSummary.countsBySourceKind.review).toBe(other.expectedReviews.filter((review) => + review.kinds.length === 0 || [4, 7, 10].includes(review.variant)).length); + const ownIds = new Set(fixture.taskIds); + expect(otherFeed.items.filter((item) => ownIds.has(item.subject.id))).toEqual([]); + expect(stable(await read(fixture, "/attention/summary"))).toEqual(stable(summary)); + + // Dismissed cards leave the count. The reviews stay stalled. + const dismissed = reviewCards.filter((item) => item.inlineResolvable).slice(0, 3); + expect(dismissed).toHaveLength(3); + await db.insert(inboxDismissals).values(dismissed.map((item) => ({ + companyId: fixture.companyId, userId: fixture.userId, itemKey: item.dismissalKey, kind: "dismiss" as const, + fingerprint: item.changeFingerprint, + }))); + const afterDismissal = await read(fixture, "/attention/summary"); + expect(afterDismissal.totalCount).toBe(summary.totalCount - 3); + expect(afterDismissal.countsBySourceKind.review).toBe(needsPerson.length - 3); + expect(afterDismissal.viewCounts.archived).toBe(summary.viewCounts.archived + 3); + const reviews = await read(fixture, "/issues?status=in_review&limit=1000"); + for (const item of dismissed) { + expect(reviews.find((row) => row.id === item.subject.id)?.reviewAttention?.state).toBe("stalled"); + } + + expect(shadow.mismatches).toEqual([]); + expect(shadow.reviewsClassified).toBeGreaterThan(0); + }, 60_000); +}); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index e8a0e7b9470..13cbda154d4 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -257,7 +257,7 @@ import { } from "./recovery/origins.js"; import { classifyIssueGraphLiveness, - classifyIssueReviewPaths, + createIssueReviewPathClassifier, type IssueGraphLivenessInput, type IssueLivenessFinding, } from "./recovery/issue-graph-liveness.js"; @@ -4992,6 +4992,7 @@ async function listIssueReviewAttentionMap( !(row.continuationRunId && listedExecutionIds.has(row.continuationRunId)), ); + const reviewIdSet = new Set(reviewIds); const recoveryPaths = [ ...(recoveryActionRows as Array<{ id: string; @@ -5012,13 +5013,13 @@ async function listIssueReviewAttentionMap( if ( recovery.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery && recovery.originId && - reviewIds.includes(recovery.originId) + reviewIdSet.has(recovery.originId) ) { recoveryPaths.push({ ...recovery, issueId: recovery.originId }); continue; } const parsed = parseIssueGraphLivenessIncidentKey(recovery.originId); - if (parsed?.companyId === companyId && reviewIds.includes(parsed.issueId)) { + if (parsed?.companyId === companyId && reviewIdSet.has(parsed.issueId)) { recoveryPaths.push({ ...recovery, issueId: parsed.issueId }); } } @@ -5118,11 +5119,10 @@ async function listIssueReviewAttentionMap( ]), ); - for (const issue of reviewIssues) { - const pathFacts = classifyIssueReviewPaths( - livenessInput, - livenessInput.issues.find((entry) => entry.id === issue.id)!, - ); + // One index of the agents and paths for every review of the request. + const reviewPathsOf = createIssueReviewPathClassifier(livenessInput); + for (const [reviewIndex, issue] of reviewIssues.entries()) { + const pathFacts = reviewPathsOf(livenessInput.issues[reviewIndex]!); const paths: IssueReviewAttentionPath[] = pathFacts.map((path) => { const interactionAudience = path.kind === "interaction" && path.ref diff --git a/server/src/services/recovery/issue-graph-liveness.ts b/server/src/services/recovery/issue-graph-liveness.ts index a40f18749eb..e606da3458b 100644 --- a/server/src/services/recovery/issue-graph-liveness.ts +++ b/server/src/services/recovery/issue-graph-liveness.ts @@ -1,4 +1,4 @@ -import { getAgentWorkEligibility, isAgentInvokable } from "@paperclipai/shared"; +import { getAgentWorkEligibility, type AgentWorkEligibility } from "@paperclipai/shared"; import { buildIssueGraphLivenessIncidentKey } from "./origins.js"; export type IssueLivenessSeverity = "warning" | "critical"; @@ -147,30 +147,96 @@ function pathEntry(issue: IssueLivenessIssueInput): IssueLivenessDependencyPathE }; } -function isInvokableAgent( - agent: IssueLivenessAgentInput | null | undefined, - agentsById: Map, -) { - return Boolean(agent && isAgentInvokable({ agent, agents: [...agentsById.values()] })); +/** + * What one classification reads about a company, built once and shared by + * every task it classifies: the agents by id, whether each can be invoked, + * the invokable agents of each company in id order, and the execution and + * waiting paths of each task. Classifying a task then costs its own paths, + * not a pass over every agent and every path of the company. + */ +interface IssueGraphLivenessIndex { + nowMs: number; + agentsById: Map; + eligibility(agent: IssueLivenessAgentInput): AgentWorkEligibility; + isInvokable(agent: IssueLivenessAgentInput | null | undefined): boolean; + invokableAgents(companyId: string): readonly IssueLivenessAgentInput[]; + activeRuns(issue: IssueLivenessIssueInput): readonly IssueLivenessExecutionPathInput[]; + queuedWakeRequests(issue: IssueLivenessIssueInput): readonly IssueLivenessExecutionPathInput[]; + pendingInteractions(issue: IssueLivenessIssueInput): readonly IssueLivenessWaitingPathInput[]; + pendingApprovals(issue: IssueLivenessIssueInput): readonly IssueLivenessWaitingPathInput[]; + externalResponseWaits(issue: IssueLivenessIssueInput): readonly IssueLivenessWaitingPathInput[]; + openRecoveryIssues(issue: IssueLivenessIssueInput): readonly IssueLivenessWaitingPathInput[]; } -function hasActiveExecutionPath( - companyId: string, - issueId: string, - activeRuns: IssueLivenessExecutionPathInput[], - queuedWakeRequests: IssueLivenessExecutionPathInput[], -) { - return [...activeRuns, ...queuedWakeRequests].some( - (entry) => entry.companyId === companyId && entry.issueId === issueId, - ); +const NO_PATHS: readonly never[] = Object.freeze([]); + +/** The paths of one task, in the order the input lists them. */ +function pathsByIssue( + paths: Path[] | undefined, +): (issue: IssueLivenessIssueInput) => readonly Path[] { + let byIssueId: Map | null = null; + return (issue) => { + if (!paths || paths.length === 0) return NO_PATHS; + if (!byIssueId) { + byIssueId = new Map(); + for (const path of paths) { + const issueId = path.issueId; + if (issueId === null) continue; + const list = byIssueId.get(issueId); + if (list) list.push(path); + else byIssueId.set(issueId, [path]); + } + } + const ofIssue = byIssueId.get(issue.id); + if (!ofIssue) return NO_PATHS; + return ofIssue.every((path) => path.companyId === issue.companyId) + ? ofIssue + : ofIssue.filter((path) => path.companyId === issue.companyId); + }; } -function hasWaitingPath( - companyId: string, - issueId: string, - waitingPaths: IssueLivenessWaitingPathInput[], -) { - return waitingPaths.some((entry) => entry.companyId === companyId && entry.issueId === issueId); +function createIssueGraphLivenessIndex(input: IssueGraphLivenessInput): IssueGraphLivenessIndex { + const agentsById = new Map(input.agents.map((agent) => [agent.id, agent])); + const agentList = [...agentsById.values()]; + // Keyed by the agent record, not its id: the input may list an id twice. + const eligibilityByAgent = new Map(); + const invokableAgentsByCompanyId = new Map(); + + function eligibility(agent: IssueLivenessAgentInput) { + let result = eligibilityByAgent.get(agent); + if (!result) { + result = getAgentWorkEligibility({ agent, agents: agentList }); + eligibilityByAgent.set(agent, result); + } + return result; + } + + function isInvokable(agent: IssueLivenessAgentInput | null | undefined) { + return Boolean(agent && eligibility(agent).invokable); + } + + return { + nowMs: readDateMs(input.now ?? new Date()) ?? Date.now(), + agentsById, + eligibility, + isInvokable, + invokableAgents(companyId) { + let result = invokableAgentsByCompanyId.get(companyId); + if (!result) { + result = input.agents + .filter((agent) => agent.companyId === companyId && isInvokable(agent)) + .sort((left, right) => left.id.localeCompare(right.id)); + invokableAgentsByCompanyId.set(companyId, result); + } + return result; + }, + activeRuns: pathsByIssue(input.activeRuns), + queuedWakeRequests: pathsByIssue(input.queuedWakeRequests), + pendingInteractions: pathsByIssue(input.pendingInteractions), + pendingApprovals: pathsByIssue(input.pendingApprovals), + externalResponseWaits: pathsByIssue(input.externalResponseWaits), + openRecoveryIssues: pathsByIssue(input.openRecoveryIssues), + }; } function readRecord(value: unknown): Record | null { @@ -225,8 +291,27 @@ export function classifyIssueReviewPaths( issue: IssueLivenessIssueInput, ): IssueReviewPathFact[] { if (issue.status !== "in_review") return []; - const nowMs = readDateMs(input.now ?? new Date()) ?? Date.now(); - const agentsById = new Map(input.agents.map((agent) => [agent.id, agent])); + return reviewPaths(createIssueGraphLivenessIndex(input), issue); +} + +/** + * Review paths for many tasks of one input. The agents and paths of the input + * are indexed once, so the cost of each task does not grow with the company. + * Each task gets the paths `classifyIssueReviewPaths` returns for it. The + * input must not change while the classifier is in use. + */ +export function createIssueReviewPathClassifier( + input: IssueGraphLivenessInput, +): (issue: IssueLivenessIssueInput) => IssueReviewPathFact[] { + const index = createIssueGraphLivenessIndex(input); + return (issue) => reviewPaths(index, issue); +} + +function reviewPaths( + index: IssueGraphLivenessIndex, + issue: IssueLivenessIssueInput, +): IssueReviewPathFact[] { + if (issue.status !== "in_review") return []; const paths: IssueReviewPathFact[] = []; if (issue.conversationAgentId && issue.conversationUserId && issue.conversationState === "waiting") { return [{ kind: "human_reviewer", ref: issue.conversationUserId, userId: issue.conversationUserId, agentId: null, since: null }]; @@ -247,8 +332,8 @@ export function classifyIssueReviewPaths( : null; const participantAgentId = readPrincipalAgentId(participant); if (participantAgentId) { - const participantAgent = agentsById.get(participantAgentId); - if (participantAgent?.companyId === issue.companyId && isInvokableAgent(participantAgent, agentsById)) { + const participantAgent = index.agentsById.get(participantAgentId); + if (participantAgent?.companyId === issue.companyId && index.isInvokable(participantAgent)) { paths.push({ kind: "execution_participant", ref: participantAgentId, @@ -268,16 +353,15 @@ export function classifyIssueReviewPaths( }); } - if (hasScheduledIssueMonitorPath(issue, nowMs)) { + if (hasScheduledIssueMonitorPath(issue, index.nowMs)) { paths.push({ kind: "monitor", ref: null, agentId: issue.assigneeAgentId ?? null, userId: null, since: null }); } const appendExecutionPaths = ( - entries: IssueLivenessExecutionPathInput[], + entries: readonly IssueLivenessExecutionPathInput[], kind: "active_run" | "queued_wake", ) => { for (const entry of entries) { - if (entry.companyId !== issue.companyId || entry.issueId !== issue.id) continue; paths.push({ kind, ref: entry.id ?? null, @@ -287,15 +371,14 @@ export function classifyIssueReviewPaths( }); } }; - appendExecutionPaths(input.activeRuns ?? [], "active_run"); - appendExecutionPaths(input.queuedWakeRequests ?? [], "queued_wake"); + appendExecutionPaths(index.activeRuns(issue), "active_run"); + appendExecutionPaths(index.queuedWakeRequests(issue), "queued_wake"); const appendWaitingPaths = ( - entries: IssueLivenessWaitingPathInput[], + entries: readonly IssueLivenessWaitingPathInput[], kind: "interaction" | "approval" | "recovery" | "external_response", ) => { for (const entry of entries) { - if (entry.companyId !== issue.companyId || entry.issueId !== issue.id) continue; paths.push({ kind, ref: entry.id ?? null, @@ -305,10 +388,10 @@ export function classifyIssueReviewPaths( }); } }; - appendWaitingPaths(input.pendingInteractions ?? [], "interaction"); - appendWaitingPaths(input.pendingApprovals ?? [], "approval"); - appendWaitingPaths(input.externalResponseWaits ?? [], "external_response"); - appendWaitingPaths(input.openRecoveryIssues ?? [], "recovery"); + appendWaitingPaths(index.pendingInteractions(issue), "interaction"); + appendWaitingPaths(index.pendingApprovals(issue), "approval"); + appendWaitingPaths(index.externalResponseWaits(issue), "external_response"); + appendWaitingPaths(index.openRecoveryIssues(issue), "recovery"); return paths; } @@ -330,15 +413,15 @@ function principalIsResolvableUser(principal: unknown): boolean { function addOwnerCandidate( candidates: IssueLivenessOwnerCandidate[], seen: Set, - agentsById: Map, + index: IssueGraphLivenessIndex, companyId: string, agentId: string | null | undefined, reason: IssueLivenessOwnerCandidateReason, sourceIssueId: string, ) { if (!agentId || seen.has(agentId)) return; - const agent = agentsById.get(agentId); - if (!agent || agent.companyId !== companyId || !isInvokableAgent(agent, agentsById)) return; + const agent = index.agentsById.get(agentId); + if (!agent || agent.companyId !== companyId || !index.isInvokable(agent)) return; seen.add(agentId); candidates.push({ agentId, reason, sourceIssueId }); } @@ -347,38 +430,27 @@ function addAgentChainCandidates( candidates: IssueLivenessOwnerCandidate[], seen: Set, startAgentId: string | null | undefined, - agentsById: Map, + index: IssueGraphLivenessIndex, companyId: string, reason: IssueLivenessOwnerCandidateReason, sourceIssueId: string, ) { const chainSeen = new Set(); - let current = startAgentId ? agentsById.get(startAgentId) : null; + let current = startAgentId ? index.agentsById.get(startAgentId) : null; while (current?.reportsTo) { if (chainSeen.has(current.reportsTo)) break; chainSeen.add(current.reportsTo); - const manager = agentsById.get(current.reportsTo); + const manager = index.agentsById.get(current.reportsTo); if (!manager || manager.companyId !== companyId) break; - addOwnerCandidate(candidates, seen, agentsById, companyId, manager.id, reason, sourceIssueId); + addOwnerCandidate(candidates, seen, index, companyId, manager.id, reason, sourceIssueId); current = manager; } } -function orderedInvokableAgents( - agents: IssueLivenessAgentInput[], - agentsById: Map, - companyId: string, -) { - return agents - .filter((agent) => agent.companyId === companyId && isInvokableAgent(agent, agentsById)) - .sort((left, right) => left.id.localeCompare(right.id)); -} - function ownerCandidatesForRecoveryIssue( issue: IssueLivenessIssueInput, - agents: IssueLivenessAgentInput[], - agentsById: Map, + index: IssueGraphLivenessIndex, options: { includeStalledAssignee?: boolean; } = {}, @@ -390,7 +462,7 @@ function ownerCandidatesForRecoveryIssue( addOwnerCandidate( candidates, seen, - agentsById, + index, issue.companyId, issue.assigneeAgentId, "stalled_blocker_assignee", @@ -402,7 +474,7 @@ function ownerCandidatesForRecoveryIssue( candidates, seen, issue.assigneeAgentId, - agentsById, + index, issue.companyId, "assignee_reporting_chain", issue.id, @@ -411,23 +483,23 @@ function ownerCandidatesForRecoveryIssue( candidates, seen, issue.createdByAgentId, - agentsById, + index, issue.companyId, "creator_reporting_chain", issue.id, ); - const invokableAgents = orderedInvokableAgents(agents, agentsById, issue.companyId); + const invokableAgents = index.invokableAgents(issue.companyId); for (const agent of invokableAgents) { if (!agent.reportsTo) { - addOwnerCandidate(candidates, seen, agentsById, issue.companyId, agent.id, "root_agent", issue.id); + addOwnerCandidate(candidates, seen, index, issue.companyId, agent.id, "root_agent", issue.id); } } for (const agent of invokableAgents) { addOwnerCandidate( candidates, seen, - agentsById, + index, issue.companyId, agent.id, "ordered_invokable_fallback", @@ -485,17 +557,12 @@ function finding(input: { } export function classifyIssueGraphLiveness(input: IssueGraphLivenessInput): IssueLivenessFinding[] { - const nowMs = readDateMs(input.now ?? new Date()) ?? Date.now(); + const index = createIssueGraphLivenessIndex(input); + const { nowMs, agentsById } = index; const issuesById = new Map(input.issues.map((issue) => [issue.id, issue])); - const agentsById = new Map(input.agents.map((agent) => [agent.id, agent])); const blockersByBlockedIssueId = new Map(); const unresolvedBlockers = new Set(); const findings: IssueLivenessFinding[] = []; - const activeRuns = input.activeRuns ?? []; - const queuedWakeRequests = input.queuedWakeRequests ?? []; - const pendingInteractions = input.pendingInteractions ?? []; - const pendingApprovals = input.pendingApprovals ?? []; - const openRecoveryIssues = input.openRecoveryIssues ?? []; for (const relation of input.relations) { const list = blockersByBlockedIssueId.get(relation.blockedIssueId) ?? []; @@ -530,10 +597,11 @@ export function classifyIssueGraphLiveness(input: IssueGraphLivenessInput): Issu function hasExplicitWaitingPath(issue: IssueLivenessIssueInput) { return Boolean(issue.assigneeUserId) || hasScheduledIssueMonitorPath(issue, nowMs) || - hasActiveExecutionPath(issue.companyId, issue.id, activeRuns, queuedWakeRequests) || - hasWaitingPath(issue.companyId, issue.id, pendingInteractions) || - hasWaitingPath(issue.companyId, issue.id, pendingApprovals) || - hasWaitingPath(issue.companyId, issue.id, openRecoveryIssues); + index.activeRuns(issue).length > 0 || + index.queuedWakeRequests(issue).length > 0 || + index.pendingInteractions(issue).length > 0 || + index.pendingApprovals(issue).length > 0 || + index.openRecoveryIssues(issue).length > 0; } function reviewFinding( @@ -542,9 +610,9 @@ export function classifyIssueGraphLiveness(input: IssueGraphLivenessInput): Issu dependencyPath: IssueLivenessIssueInput[], ): IssueLivenessFinding | null { if (reviewIssue.status !== "in_review") return null; - if (classifyIssueReviewPaths(input, reviewIssue).length > 0) return null; + if (reviewPaths(index, reviewIssue).length > 0) return null; - const ownerCandidates = ownerCandidatesForRecoveryIssue(reviewIssue, input.agents, agentsById, { + const ownerCandidates = ownerCandidatesForRecoveryIssue(reviewIssue, index, { includeStalledAssignee: true, }); @@ -555,7 +623,7 @@ export function classifyIssueGraphLiveness(input: IssueGraphLivenessInput): Issu const participantAgentId = readPrincipalAgentId(participant); if (participantAgentId) { const participantAgent = agentsById.get(participantAgentId); - if (isInvokableAgent(participantAgent, agentsById) && participantAgent?.companyId === reviewIssue.companyId) return null; + if (index.isInvokable(participantAgent) && participantAgent?.companyId === reviewIssue.companyId) return null; return finding({ issue: source, @@ -610,7 +678,7 @@ export function classifyIssueGraphLiveness(input: IssueGraphLivenessInput): Issu blocker: IssueLivenessIssueInput, dependencyPath: IssueLivenessIssueInput[], ): IssueLivenessFinding | null { - const ownerCandidates = ownerCandidatesForRecoveryIssue(blocker, input.agents, agentsById, { + const ownerCandidates = ownerCandidatesForRecoveryIssue(blocker, index, { includeStalledAssignee: true, }); @@ -669,7 +737,7 @@ export function classifyIssueGraphLiveness(input: IssueGraphLivenessInput): Issu const blockerAgent = agentsById.get(blocker.assigneeAgentId); const blockerEligibility = blockerAgent - ? getAgentWorkEligibility({ agent: blockerAgent, agents: input.agents }) + ? index.eligibility(blockerAgent) : null; if (!blockerAgent || blockerAgent.companyId !== source.companyId || !blockerEligibility?.invokable) { return finding({