diff --git a/server/src/__tests__/agent-live-run-routes.test.ts b/server/src/__tests__/agent-live-run-routes.test.ts index 227245c63f7..3b3013dc68f 100644 --- a/server/src/__tests__/agent-live-run-routes.test.ts +++ b/server/src/__tests__/agent-live-run-routes.test.ts @@ -11,6 +11,7 @@ const mockAgentService = vi.hoisted(() => ({ const mockHeartbeatService = vi.hoisted(() => ({ list: vi.fn(async () => []), buildRunOutputSilence: vi.fn(), + buildRunOutputSilences: vi.fn(), decorateActiveRunStatus: vi.fn(), getRunIssueSummary: vi.fn(), getActiveRunIssueSummaryForAgent: vi.fn(), @@ -344,6 +345,7 @@ describe("agent live run routes", () => { }); mockInstanceSettingsService.listCompanyIds.mockResolvedValue(["company-1"]); mockHeartbeatService.buildRunOutputSilence.mockResolvedValue(null); + mockHeartbeatService.buildRunOutputSilences.mockImplementation(async (runs: unknown[]) => runs.map(() => null)); mockHeartbeatService.decorateActiveRunStatus.mockImplementation((run) => ({ ...run, currentStatusMessage: null, @@ -838,9 +840,9 @@ describe("agent live run routes", () => { expect(res.body).toHaveLength(50); expect(mockRunSecretRedactionRegistry.redactForRuns).toHaveBeenCalledTimes(1); expect(mockRunSecretRedactionRegistry.redactForRun).not.toHaveBeenCalled(); - expect(mockHeartbeatService.buildRunOutputSilence).toHaveBeenCalledTimes( - 50, - ); + expect(mockHeartbeatService.buildRunOutputSilence).not.toHaveBeenCalled(); + expect(mockHeartbeatService.buildRunOutputSilences).toHaveBeenCalledTimes(1); + expect(mockHeartbeatService.buildRunOutputSilences.mock.calls[0]![0]).toHaveLength(50); }); it("treats explicit zero or invalid live run limit as the capped default", async () => { diff --git a/server/src/__tests__/live-runs-batched-reads.test.ts b/server/src/__tests__/live-runs-batched-reads.test.ts new file mode 100644 index 00000000000..a51b56c1f21 --- /dev/null +++ b/server/src/__tests__/live-runs-batched-reads.test.ts @@ -0,0 +1,245 @@ +import { randomUUID } from "node:crypto"; +import express from "express"; +import request from "supertest"; +import { eq } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { afterAll, afterEach, beforeAll, expect, it, vi } from "vitest"; +import { heartbeatRunWatchdogDecisions, heartbeatRuns, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; +import { agentRoutes } from "../routes/agents.js"; +import { errorHandler } from "../middleware/index.js"; +import { authorizationService, type AuthorizationActor } from "../services/authorization.js"; +import { + canActorReadHeartbeatRun, getHeartbeatRunReadDecisions, redactHeartbeatRunListRow, +} from "../services/heartbeat-run-privacy.js"; +import { + ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS, ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS, recoveryService, +} from "../services/recovery/service.js"; +import { describeEmbeddedPostgres, useEmbeddedPostgres } from "./helpers/route-test-harness.js"; +import { + ISSUE_READ_VISIBILITY_ACTORS, seedIssueReadVisibilityFixture, type IssueReadVisibilityActorName, + type IssueReadVisibilityFixture, +} from "./helpers/issue-read-visibility-fixture.js"; + +const NOW = new Date("2026-11-13T12:00:00.000Z"); +const HOUR = 60 * 60 * 1000; + +type RunBinding = { id: string; companyId: string; scopeKind: "company" | "issue"; issueId: string | null }; + +describeEmbeddedPostgres("live runs read in a bounded number of statements", () => { + const ctx = useEmbeddedPostgres("paperclip-live-runs-batch-"); + const oldPrivacy = process.env.PAPERCLIP_ISSUE_PRIVACY_MODE; + let f: IssueReadVisibilityFixture; + let runs: RunBinding[]; + let foreignRuns: RunBinding[]; + let silenceRunIds: Record<"noOutput" | "snoozed" | "expiredSnooze" | "dismissed" | "recent" | "evaluated" | "doneEvaluation" | "queued", string>; + + let order = 0; + async function insertRun(input: { + companyId: string; agentId: string; issueId?: string | null; scopeKind?: "company" | "issue"; + status?: string; lastOutputAt?: Date | null; lastOutputSeq?: number; lastOutputStream?: string | null; + }) { + const id = randomUUID(); + order += 1; + const scopeKind = input.scopeKind ?? (input.issueId ? "issue" : "company"); + await ctx.db.insert(heartbeatRuns).values({ + id, companyId: input.companyId, agentId: input.agentId, scopeKind, issueId: input.issueId ?? null, + status: input.status ?? "running", startedAt: new Date(NOW.getTime() - 3 * HOUR), + createdAt: new Date(NOW.getTime() - 3 * HOUR - order * 1000), lastOutputAt: input.lastOutputAt ?? null, + lastOutputSeq: input.lastOutputSeq ?? 0, lastOutputStream: input.lastOutputStream ?? null, + usageJson: { inputTokens: 11, cachedInputTokens: 2, outputTokens: 7 }, resultJson: { costUsd: 0.42, summary: "secret summary" }, + contextSnapshot: input.issueId ? { issueId: input.issueId } : {}, + }); + return { id, companyId: input.companyId, scopeKind, issueId: input.issueId ?? null } satisfies RunBinding; + } + + async function seedRuns(fixture: IssueReadVisibilityFixture, openTaskRuns: number) { + const agentIds = [fixture.readerAgentId, fixture.assigneeAgentId, fixture.lowTrustAgentId]; + const taskIds = [ + ...Array.from({ length: openTaskRuns }, (_, index) => fixture.publicIds[index % fixture.publicIds.length]!), + ...fixture.ownedPrivateIds, ...fixture.grantedPrivateIds, ...fixture.assignedPrivateIds, ...fixture.unreadablePrivateIds, + ...fixture.privateProjectIssueIds, fixture.hiddenId, + ]; + const seeded: RunBinding[] = []; + for (const [index, issueId] of taskIds.entries()) { + seeded.push(await insertRun({ companyId: fixture.companyId, agentId: agentIds[index % agentIds.length]!, issueId })); + } + seeded.push(await insertRun({ companyId: fixture.companyId, agentId: fixture.readerAgentId })); + const doomedTaskId = randomUUID(); + await ctx.db.insert(issues).values({ id: doomedTaskId, companyId: fixture.companyId, title: "Deleted", status: "todo" }); + const orphan = await insertRun({ companyId: fixture.companyId, agentId: fixture.assigneeAgentId, issueId: doomedTaskId }); + await ctx.db.delete(issues).where(eq(issues.id, doomedTaskId)); + seeded.push({ ...orphan, issueId: null }); + return seeded; + } + + beforeAll(async () => { + process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = "enforce"; + f = await seedIssueReadVisibilityFixture(ctx.db, { publicCount: 12 }); + runs = await seedRuns(f, 3); + foreignRuns = [ + await insertRun({ companyId: f.foreignCompanyId, agentId: f.foreignAgentId, issueId: f.foreignIds[0]! }), + await insertRun({ companyId: f.foreignCompanyId, agentId: f.foreignAgentId, issueId: f.foreignIds[1]! }), + await insertRun({ companyId: f.foreignCompanyId, agentId: f.foreignAgentId }), + ]; + const silenceRun = (extra: Partial[0]>) => + insertRun({ companyId: f.companyId, agentId: f.readerAgentId, ...extra }); + silenceRunIds = { + noOutput: (await silenceRun({})).id, + snoozed: (await silenceRun({ lastOutputAt: new Date(NOW.getTime() - 2 * HOUR), lastOutputSeq: 4, lastOutputStream: "stdout" })).id, + expiredSnooze: (await silenceRun({ lastOutputAt: new Date(NOW.getTime() - 2 * HOUR), lastOutputSeq: 2, lastOutputStream: "stderr" })).id, + dismissed: (await silenceRun({ lastOutputAt: new Date(NOW.getTime() - 5 * HOUR), lastOutputSeq: 9, lastOutputStream: "other" })).id, + recent: (await silenceRun({ lastOutputAt: new Date(NOW.getTime() - 10 * 60 * 1000), lastOutputSeq: 1, lastOutputStream: "stdout" })).id, + evaluated: (await silenceRun({})).id, + doneEvaluation: (await silenceRun({})).id, + queued: (await silenceRun({ status: "queued" })).id, + }; + await ctx.db.insert(heartbeatRunWatchdogDecisions).values([ + { companyId: f.companyId, runId: silenceRunIds.snoozed, decision: "snooze", snoozedUntil: new Date(NOW.getTime() + HOUR), createdAt: new Date(NOW.getTime() - 2 * HOUR) }, + { companyId: f.companyId, runId: silenceRunIds.snoozed, decision: "continue", snoozedUntil: new Date(NOW.getTime() + 2 * HOUR), createdAt: new Date(NOW.getTime() - HOUR) }, + { companyId: f.companyId, runId: silenceRunIds.expiredSnooze, decision: "snooze", snoozedUntil: new Date(NOW.getTime() - HOUR) }, + { companyId: f.companyId, runId: silenceRunIds.dismissed, decision: "dismissed_false_positive" }, + { companyId: f.companyId, runId: silenceRunIds.dismissed, decision: "snooze", snoozedUntil: new Date(NOW.getTime() + HOUR) }, + ]); + const evaluationId = randomUUID(); + await ctx.db.insert(issues).values([ + { id: evaluationId, companyId: f.companyId, title: "Evaluate", status: "todo", identifier: "EVAL-1", + assigneeAgentId: f.assigneeAgentId, originKind: "stale_active_run_evaluation", originId: silenceRunIds.evaluated }, + { companyId: f.companyId, title: "Evaluated before", status: "done", originKind: "stale_active_run_evaluation", originId: silenceRunIds.doneEvaluation }, + { companyId: f.foreignCompanyId, title: "Foreign evaluation", status: "todo", originKind: "stale_active_run_evaluation", originId: silenceRunIds.noOutput }, + ]); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(() => { + if (oldPrivacy === undefined) delete process.env.PAPERCLIP_ISSUE_PRIVACY_MODE; + else process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = oldPrivacy; + }); + afterEach(() => { vi.useRealTimers(); }); + + function app(actor: AuthorizationActor, db = ctx.db) { + const result = express(); + result.use(express.json()); + result.use((req, _res, next) => { req.actor = { ...actor } as Express.Request["actor"]; next(); }); + result.use("/api", agentRoutes(db, {} as never)); + result.use(errorHandler); + return result; + } + function recording() { + const statements: string[] = []; + const db = drizzle(ctx.db.$client, { + schema: ctx.db._.fullSchema, logger: { logQuery: (query) => statements.push(query) }, + }) as unknown as typeof ctx.db; + return { db, statements }; + } + async function liveRuns(actor: AuthorizationActor, companyId = f.companyId, db = ctx.db) { + return request(app(actor, db)).get(`/api/companies/${companyId}/live-runs`); + } + const readEach = (actor: AuthorizationActor, bindings: RunBinding[]) => + Promise.all(bindings.map((run) => canActorReadHeartbeatRun(ctx.db, authorizationService(ctx.db), { ...actor }, run))); + + it.each(ISSUE_READ_VISIBILITY_ACTORS)("decides every run read for %s exactly as one check per run does", async (name) => { + const actor = f.actors[name]; + const bindings = [...runs, ...foreignRuns, ...foreignRuns.map((run) => ({ ...run, companyId: f.companyId }))]; + const batched = await getHeartbeatRunReadDecisions(ctx.db, authorizationService(ctx.db), { ...actor }, bindings); + expect(batched).toEqual(await readEach(actor, bindings)); + }); + + it("covers readable and unreadable runs in the fixture", async () => { + const decisions = async (name: IssueReadVisibilityActorName) => readEach(f.actors[name], runs); + expect((await decisions("instance admin")).filter(Boolean).length).toBeGreaterThan((await decisions("board member")).filter(Boolean).length); + expect(await decisions("board member")).toContain(false); + expect(await decisions("board member")).toContain(true); + expect(await decisions("agent key")).toContain(false); + expect(runs.at(-1)).toMatchObject({ scopeKind: "issue", issueId: null }); + expect(await readEach(f.actors["instance admin"], [runs.at(-1)!])).toEqual([false]); + }); + + it.each(ISSUE_READ_VISIBILITY_ACTORS)("returns %s the live runs one check per run would, redacting the rest field by field", async (name) => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(NOW); + const actor = f.actors[name]; + const res = await liveRuns(actor); + if (res.status !== 200) { + expect([401, 403]).toContain(res.status); + for (const run of runs) expect(JSON.stringify(res.body)).not.toContain(run.id); + return; + } + const full = await liveRuns(f.actors["local implicit board"]); + expect(full.status).toBe(200); + const fullById = new Map((full.body as Array & { id: string }>).map((row) => [row.id, row])); + const body = res.body as Array & { id: string; scopeKind?: string }>; + expect(body.map((row) => row.id)).toEqual([...fullById.keys()]); + const bindings = body.map((row) => { + const source = fullById.get(row.id)!; + return { id: row.id, companyId: f.companyId, scopeKind: source.scopeKind as "company" | "issue", issueId: (source.issueId as string | null) ?? null }; + }); + const expected = await readEach(actor, bindings); + for (const [index, row] of body.entries()) { + const source = fullById.get(row.id)!; + if (expected[index]) { + expect(row.redacted).toBeUndefined(); + expect(row.outputSilence).toEqual(source.outputSilence); + expect(row.issueId).toEqual(source.issueId); + } else { + expect(row).toEqual(JSON.parse(JSON.stringify(redactHeartbeatRunListRow(source)))); + } + } + expect(expected).toContain(true); + }); + + it("builds each run's output silence as the single-run reader does, including runs with no output yet", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(NOW); + const res = await liveRuns(f.actors["local implicit board"]); + expect(res.status).toBe(200); + const byId = new Map((res.body as Array<{ id: string; outputSilence: unknown }>).map((row) => [row.id, row.outputSilence])); + const recovery = recoveryService(ctx.db, { enqueueWakeup: vi.fn() } as never); + const rows = await ctx.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + const single = new Map(await Promise.all(rows.map(async (run) => [run.id, JSON.parse(JSON.stringify(await recovery.buildRunOutputSilence(run, NOW)))] as const))); + for (const [id, silence] of byId) { + if (silence !== undefined) expect(silence).toEqual(single.get(id)); + } + const thresholds = { suspicionThresholdMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS, criticalThresholdMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS }; + const silenceOf = (id: string) => single.get(id) as Record; + expect(silenceOf(silenceRunIds.noOutput)).toEqual({ + lastOutputAt: null, lastOutputSeq: 0, lastOutputStream: null, + silenceStartedAt: new Date(NOW.getTime() - 3 * HOUR).toISOString(), silenceAgeMs: 3 * HOUR, level: "critical", ...thresholds, + snoozedUntil: null, evaluationIssueId: null, evaluationIssueIdentifier: null, evaluationIssueAssigneeAgentId: null, + }); + expect(silenceOf(silenceRunIds.snoozed)).toMatchObject({ + lastOutputSeq: 4, lastOutputStream: "stdout", level: "snoozed", snoozedUntil: new Date(NOW.getTime() + 2 * HOUR).toISOString(), + }); + expect(silenceOf(silenceRunIds.expiredSnooze)).toMatchObject({ lastOutputStream: "stderr", level: "critical", snoozedUntil: null }); + expect(silenceOf(silenceRunIds.dismissed)).toMatchObject({ lastOutputStream: null, level: "not_applicable", snoozedUntil: null }); + expect(silenceOf(silenceRunIds.recent)).toMatchObject({ level: "suspicious", silenceAgeMs: 10 * 60 * 1000 }); + expect(silenceOf(silenceRunIds.evaluated)).toMatchObject({ evaluationIssueIdentifier: "EVAL-1", evaluationIssueAssigneeAgentId: f.assigneeAgentId }); + expect(silenceOf(silenceRunIds.doneEvaluation)).toMatchObject({ evaluationIssueId: null }); + expect(silenceOf(silenceRunIds.queued)).toMatchObject({ level: "not_applicable", silenceAgeMs: null }); + + const batched = await recovery.buildRunOutputSilences([...rows, ...rows.slice(0, 2)], NOW); + expect(batched.map((silence) => JSON.parse(JSON.stringify(silence)))) + .toEqual([...rows, ...rows.slice(0, 2)].map((run) => single.get(run.id))); + expect(await recovery.buildRunOutputSilences([], NOW)).toEqual([]); + const missing = rows.find((run) => run.id === silenceRunIds.noOutput)!; + expect(JSON.parse(JSON.stringify(await recovery.buildRunOutputSilence({ ...missing, id: randomUUID() }, NOW)))) + .toEqual(silenceOf(silenceRunIds.noOutput)); + }); + + it.each([ + "board owner", "board member", "instance admin", "local implicit board", "agent key", "agent run for operator", + ] as const)("sends the same number of statements for %s whether 19 or 49 runs are live", async (name) => { + const small = await seedIssueReadVisibilityFixture(ctx.db, { publicCount: 12 }); + const large = await seedIssueReadVisibilityFixture(ctx.db, { publicCount: 12 }); + const smallRuns = await seedRuns(small, 0); + const largeRuns = await seedRuns(large, 30); + const actorFor = (fixture: IssueReadVisibilityFixture) => fixture.actors[name]; + const first = recording(); + const smallRes = await liveRuns(actorFor(small), small.companyId, first.db); + const second = recording(); + const largeRes = await liveRuns(actorFor(large), large.companyId, second.db); + expect(smallRes.status).toBe(200); + expect(largeRes.status).toBe(200); + expect(smallRes.body).toHaveLength(smallRuns.length + 1); + expect(largeRes.body).toHaveLength(largeRuns.length + 1); + expect(second.statements.length).toBe(first.statements.length); + expect(second.statements.length).toBeLessThanOrEqual(24); + }); +}); diff --git a/server/src/modules/active-run-watchdog/adapters/postgres.ts b/server/src/modules/active-run-watchdog/adapters/postgres.ts index a4f810932b9..f7e01ce56ca 100644 --- a/server/src/modules/active-run-watchdog/adapters/postgres.ts +++ b/server/src/modules/active-run-watchdog/adapters/postgres.ts @@ -31,7 +31,7 @@ import { import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js"; import { RECOVERY_ORIGIN_KINDS } from "../../../services/recovery/origins.js"; import { isTerminalIssueStatus } from "../domain/policy.js"; -import type { WatchdogRunReader, WatchdogWriter } from "../application/ports.js"; +import type { FindLatestDecisionResult, WatchdogRunReader, WatchdogWriter } from "../application/ports.js"; import type { EvaluationIssueSnapshot, FoldOutcome, @@ -159,10 +159,14 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch return row?.pending ?? true; } - async function findLatestDecision(companyId: string, runId: string, now: Date) { + async function findLatestDecisions(companyId: string, runIds: readonly string[], now: Date) { + const ids = [...new Set(runIds)]; + const result = new Map(); + if (ids.length === 0) return result; const [quietUntilRows, dismissedRows] = await Promise.all([ db - .select({ + .selectDistinctOn([heartbeatRunWatchdogDecisions.runId], { + runId: heartbeatRunWatchdogDecisions.runId, decision: heartbeatRunWatchdogDecisions.decision, snoozedUntil: heartbeatRunWatchdogDecisions.snoozedUntil, }) @@ -170,32 +174,39 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch .where( and( eq(heartbeatRunWatchdogDecisions.companyId, companyId), - eq(heartbeatRunWatchdogDecisions.runId, runId), + inArray(heartbeatRunWatchdogDecisions.runId, ids), inArray(heartbeatRunWatchdogDecisions.decision, ["snooze", "continue"]), gt(heartbeatRunWatchdogDecisions.snoozedUntil, now), ), ) - .orderBy(desc(heartbeatRunWatchdogDecisions.createdAt)) - .limit(1), + .orderBy(heartbeatRunWatchdogDecisions.runId, desc(heartbeatRunWatchdogDecisions.createdAt)), db - .select({ id: heartbeatRunWatchdogDecisions.id }) + .selectDistinct({ runId: heartbeatRunWatchdogDecisions.runId }) .from(heartbeatRunWatchdogDecisions) .where( and( eq(heartbeatRunWatchdogDecisions.companyId, companyId), - eq(heartbeatRunWatchdogDecisions.runId, runId), + inArray(heartbeatRunWatchdogDecisions.runId, ids), eq(heartbeatRunWatchdogDecisions.decision, "dismissed_false_positive"), ), - ) - .limit(1), + ), ]); - const quietUntilRow = quietUntilRows[0]; - return { - dismissedFalsePositive: dismissedRows.length > 0, - quietUntilDecision: quietUntilRow && quietUntilRow.snoozedUntil - ? { decision: quietUntilRow.decision as "snooze" | "continue", snoozedUntil: quietUntilRow.snoozedUntil } - : null, - }; + const quietUntilByRun = new Map(quietUntilRows.map((row) => [row.runId, row])); + const dismissed = new Set(dismissedRows.map((row) => row.runId)); + for (const runId of ids) { + const quietUntilRow = quietUntilByRun.get(runId); + result.set(runId, { + dismissedFalsePositive: dismissed.has(runId), + quietUntilDecision: quietUntilRow && quietUntilRow.snoozedUntil + ? { decision: quietUntilRow.decision as "snooze" | "continue", snoozedUntil: quietUntilRow.snoozedUntil } + : null, + }); + } + return result; + } + + async function findLatestDecision(companyId: string, runId: string, now: Date): Promise { + return (await findLatestDecisions(companyId, [runId], now)).get(runId)!; } function selectEvaluationIssueSnapshot() { @@ -211,21 +222,29 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch }; } - async function findOpenStaleRunEvaluation(companyId: string, runId: string): Promise { - const [row] = await db - .select(selectEvaluationIssueSnapshot()) + async function findOpenStaleRunEvaluations(companyId: string, runIds: readonly string[]) { + const ids = [...new Set(runIds)]; + const result = new Map(); + if (ids.length === 0) return result; + const rows = await db + .selectDistinctOn([issues.originId], selectEvaluationIssueSnapshot()) .from(issues) .where( and( eq(issues.companyId, companyId), eq(issues.originKind, STALE_ACTIVE_RUN_EVALUATION_ORIGIN_KIND), - eq(issues.originId, runId), + inArray(issues.originId, ids), visibleIssueCondition(), notInArray(issues.status, ["done", "cancelled"]), ), ) - .limit(1); - return row ?? null; + .orderBy(issues.originId); + for (const row of rows) if (row.originId) result.set(row.originId, row); + return result; + } + + async function findOpenStaleRunEvaluation(companyId: string, runId: string): Promise { + return (await findOpenStaleRunEvaluations(companyId, [runId])).get(runId) ?? null; } async function findEvaluationIssueById(companyId: string, issueId: string): Promise { @@ -532,7 +551,9 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch findRunForCompany, isRuntimeSettlementPending, findLatestDecision, + findLatestDecisions, findOpenStaleRunEvaluation, + findOpenStaleRunEvaluations, findEvaluationIssueById, findLatestSameRunTerminalEvidence, findSourceIssue, diff --git a/server/src/modules/active-run-watchdog/application/ports.ts b/server/src/modules/active-run-watchdog/application/ports.ts index d574d196c02..18e71b2f4b4 100644 --- a/server/src/modules/active-run-watchdog/application/ports.ts +++ b/server/src/modules/active-run-watchdog/application/ports.ts @@ -56,7 +56,9 @@ export interface WatchdogRunReader { findRunForCompany(companyId: string, runId: string): Promise; isRuntimeSettlementPending(companyId: string, runId: string): Promise; findLatestDecision(companyId: string, runId: string, now: Date): Promise; + findLatestDecisions(companyId: string, runIds: readonly string[], now: Date): Promise>; findOpenStaleRunEvaluation(companyId: string, runId: string): Promise; + findOpenStaleRunEvaluations(companyId: string, runIds: readonly string[]): Promise>; findLatestSameRunTerminalEvidence( companyId: string, input: FindLatestSameRunTerminalEvidenceInput, diff --git a/server/src/modules/active-run-watchdog/application/use-cases.test.ts b/server/src/modules/active-run-watchdog/application/use-cases.test.ts index 2b0f1f1e268..48c0a541561 100644 --- a/server/src/modules/active-run-watchdog/application/use-cases.test.ts +++ b/server/src/modules/active-run-watchdog/application/use-cases.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { createBuildRunOutputSilence, + createBuildRunOutputSilences, createFoldSourceResolvedRun, createRecordWatchdogDecision, createScanSilentActiveRuns, @@ -35,18 +36,29 @@ describe("application", () => { } function makeReader(overrides: Partial = {}): WatchdogRunReader { - return { + const reader: WatchdogRunReader = { findCandidateSilentRuns: async () => [], findRunForCompany: async () => null, isRuntimeSettlementPending: async () => false, findLatestDecision: async () => ({ quietUntilDecision: null, dismissedFalsePositive: false }), + findLatestDecisions: async (companyId, runIds, now) => new Map(await Promise.all(runIds.map(async (runId) => + [runId, await reader.findLatestDecision(companyId, runId, now)] as const))), findOpenStaleRunEvaluation: async () => null, + findOpenStaleRunEvaluations: async (companyId, runIds) => { + const found = new Map(); + for (const runId of runIds) { + const evaluation = await reader.findOpenStaleRunEvaluation(companyId, runId); + if (evaluation) found.set(runId, evaluation); + } + return found; + }, findLatestSameRunTerminalEvidence: async () => null, findSourceIssue: async () => null, findRunningAgent: async () => null, findEvaluationIssueById: async () => null, ...overrides, }; + return reader; } describe("createBuildRunOutputSilence", () => { @@ -105,6 +117,33 @@ describe("application", () => { expect(summary).toMatchObject({ level: "snoozed", snoozedUntil }); }); + it("reads decisions and evaluations once per company for a batch, defaulting runs the reader omits", async () => { + const now = new Date("2026-01-01T05:00:00.000Z"); + const calls: string[] = []; + const reader = makeReader({ + findLatestDecisions: async (companyId, runIds) => { + calls.push(`decisions ${companyId} ${runIds.join(",")}`); + return new Map(); + }, + findOpenStaleRunEvaluations: async (companyId, runIds) => { + calls.push(`evaluations ${companyId} ${runIds.join(",")}`); + return new Map(); + }, + }); + const deps = { reader, suspicionThresholdMs: SUSPICION_THRESHOLD_MS, criticalThresholdMs: CRITICAL_THRESHOLD_MS }; + const runs = [makeRun({ id: "run-1" }), makeRun({ id: "run-2", companyId: "company-2" }), makeRun({ id: "run-3" })]; + + const summaries = await createBuildRunOutputSilences(deps)(runs, now); + + expect(calls.sort()).toEqual([ + "decisions company-1 run-1,run-3", "decisions company-2 run-2", + "evaluations company-1 run-1,run-3", "evaluations company-2 run-2", + ]); + const single = await createBuildRunOutputSilence({ ...deps, reader: makeReader() })(makeRun(), now); + expect(summaries).toEqual([single, single, single]); + expect(single).toMatchObject({ level: "critical", snoozedUntil: null, evaluationIssueId: null, lastOutputAt: null, lastOutputSeq: 0 }); + }); + it("reports not_applicable for a run that is not running", async () => { const now = new Date("2026-01-01T05:00:00.000Z"); const run = makeRun({ status: "succeeded" }); diff --git a/server/src/modules/active-run-watchdog/application/use-cases.ts b/server/src/modules/active-run-watchdog/application/use-cases.ts index 9ea0bf24d88..cbdff43d0be 100644 --- a/server/src/modules/active-run-watchdog/application/use-cases.ts +++ b/server/src/modules/active-run-watchdog/application/use-cases.ts @@ -31,43 +31,68 @@ export type BuildRunOutputSilenceRunInput = Pick< "id" | "companyId" | "status" | "lastOutputAt" | "lastOutputSeq" | "lastOutputStream" | "processStartedAt" | "startedAt" | "createdAt" >; +export function createBuildRunOutputSilences(deps: BuildRunOutputSilenceDeps) { + return async function buildRunOutputSilences( + runs: readonly BuildRunOutputSilenceRunInput[], + now: Date, + ): Promise { + const runIdsByCompany = new Map(); + for (const run of runs) { + const runIds = runIdsByCompany.get(run.companyId) ?? []; + runIds.push(run.id); + runIdsByCompany.set(run.companyId, runIds); + } + const stateByCompany = new Map(await Promise.all([...runIdsByCompany].map(async ([companyId, runIds]) => { + const [decisions, evaluations] = await Promise.all([ + deps.reader.findLatestDecisions(companyId, runIds, now), + deps.reader.findOpenStaleRunEvaluations(companyId, runIds), + ]); + return [companyId, { decisions, evaluations }] as const; + }))); + return runs.map((run) => { + const state = stateByCompany.get(run.companyId); + const { dismissedFalsePositive, quietUntilDecision } = state?.decisions.get(run.id) + ?? { dismissedFalsePositive: false, quietUntilDecision: null }; + const evaluation = state?.evaluations.get(run.id) ?? null; + const isRunningRun = run.status === "running"; + const silenceAgeMsValue = isRunningRun ? silenceAgeMs(run, now) : null; + const level = classifySilenceLevel({ + isRunningRun, + silenceAgeMs: silenceAgeMsValue, + dismissedFalsePositive, + snoozed: Boolean(quietUntilDecision), + suspicionThresholdMs: deps.suspicionThresholdMs, + criticalThresholdMs: deps.criticalThresholdMs, + }); + + return { + lastOutputAt: run.lastOutputAt ?? null, + lastOutputSeq: run.lastOutputSeq ?? 0, + lastOutputStream: run.lastOutputStream === "stdout" || run.lastOutputStream === "stderr" + ? run.lastOutputStream + : null, + silenceStartedAt: silenceStartedAt(run), + silenceAgeMs: silenceAgeMsValue, + level, + suspicionThresholdMs: deps.suspicionThresholdMs, + criticalThresholdMs: deps.criticalThresholdMs, + snoozedUntil: dismissedFalsePositive ? null : quietUntilDecision?.snoozedUntil ?? null, + evaluationIssueId: evaluation?.id ?? null, + evaluationIssueIdentifier: evaluation?.identifier ?? null, + evaluationIssueAssigneeAgentId: evaluation?.assigneeAgentId ?? null, + }; + }); + }; +} + export function createBuildRunOutputSilence(deps: BuildRunOutputSilenceDeps) { + const buildRunOutputSilences = createBuildRunOutputSilences(deps); return async function buildRunOutputSilence( run: BuildRunOutputSilenceRunInput, now: Date, ): Promise { - const [decisionState, evaluation] = await Promise.all([ - deps.reader.findLatestDecision(run.companyId, run.id, now), - deps.reader.findOpenStaleRunEvaluation(run.companyId, run.id), - ]); - const { dismissedFalsePositive, quietUntilDecision } = decisionState; - const isRunningRun = run.status === "running"; - const silenceAgeMsValue = isRunningRun ? silenceAgeMs(run, now) : null; - const level = classifySilenceLevel({ - isRunningRun, - silenceAgeMs: silenceAgeMsValue, - dismissedFalsePositive, - snoozed: Boolean(quietUntilDecision), - suspicionThresholdMs: deps.suspicionThresholdMs, - criticalThresholdMs: deps.criticalThresholdMs, - }); - - return { - lastOutputAt: run.lastOutputAt ?? null, - lastOutputSeq: run.lastOutputSeq ?? 0, - lastOutputStream: run.lastOutputStream === "stdout" || run.lastOutputStream === "stderr" - ? run.lastOutputStream - : null, - silenceStartedAt: silenceStartedAt(run), - silenceAgeMs: silenceAgeMsValue, - level, - suspicionThresholdMs: deps.suspicionThresholdMs, - criticalThresholdMs: deps.criticalThresholdMs, - snoozedUntil: dismissedFalsePositive ? null : quietUntilDecision?.snoozedUntil ?? null, - evaluationIssueId: evaluation?.id ?? null, - evaluationIssueIdentifier: evaluation?.identifier ?? null, - evaluationIssueAssigneeAgentId: evaluation?.assigneeAgentId ?? null, - }; + const [summary] = await buildRunOutputSilences([run], now); + return summary!; }; } diff --git a/server/src/modules/active-run-watchdog/index.ts b/server/src/modules/active-run-watchdog/index.ts index 92a1c9fe779..07a0adee1d4 100644 --- a/server/src/modules/active-run-watchdog/index.ts +++ b/server/src/modules/active-run-watchdog/index.ts @@ -4,6 +4,7 @@ import { createPostgresWatchdogAdapter } from "./adapters/postgres.js"; import { createProcessAdapter } from "./adapters/process.js"; import { createBuildRunOutputSilence, + createBuildRunOutputSilences, createFoldSourceResolvedRun, createRecordWatchdogDecision, createScanSilentActiveRuns, @@ -42,6 +43,11 @@ export function createActiveRunWatchdog(db: Db, config: ActiveRunWatchdogConfig) suspicionThresholdMs: config.suspicionThresholdMs, criticalThresholdMs: config.criticalThresholdMs, }), + buildRunOutputSilences: createBuildRunOutputSilences({ + reader: postgresAdapter, + suspicionThresholdMs: config.suspicionThresholdMs, + criticalThresholdMs: config.criticalThresholdMs, + }), scanSilentActiveRuns, recordWatchdogDecision: createRecordWatchdogDecision({ reader: postgresAdapter, diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index eb8be845b59..e47f09a20c8 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -7304,13 +7304,17 @@ export function agentRoutes( } const projections = await executionProjectionsForRuns(db, companyId, rows.map(run => run.id)); - res.json(await runRedactions.redactForRuns(companyId, await Promise.all(rows.map(async (run) => !(await actorCanReadRun(req, run)) ? redactHeartbeatRunListRow(run) : ({ + const readDecisions = await getHeartbeatRunReadDecisions(db, access, req.actor, rows); + const readableRuns = rows.filter((_, index) => readDecisions[index]); + const silences = await heartbeat.buildRunOutputSilences(readableRuns); + const silenceByRun = new Map(readableRuns.map((run, index) => [run.id, silences[index]!])); + res.json(await runRedactions.redactForRuns(companyId, rows.map((run, index) => !readDecisions[index] ? redactHeartbeatRunListRow(run) : ({ ...heartbeat.decorateActiveRunStatus(run), agentAppearance: resolveAgentAppearance(run.agentAppearance, run.agentId), avatarUrl: agentAvatarUrl(resolveAgentAppearance(run.agentAppearance, run.agentId), 512), execution: projections.get(run.id) ?? null, - outputSilence: await heartbeat.buildRunOutputSilence(run), - }))), req.actor.type === "board" ? getActorInfo(req).actorId : null)); + outputSilence: silenceByRun.get(run.id)!, + })), req.actor.type === "board" ? getActorInfo(req).actorId : null)); }); function readHeartbeatRunId(req: Request): string { diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 64d5ad5804b..23bfa9766c1 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1774,6 +1774,24 @@ export function heartbeatService( return recovery.buildRunOutputSilence(run, now); } + async function buildRunOutputSilences( + runs: ReadonlyArray>, + now = new Date(), + ) { + return recovery.buildRunOutputSilences(runs, now); + } + async function reconcileResolvedDependencyWakes(opts?: Parameters[0]) { return recovery.reconcileResolvedDependencyWakeBackstop(opts); } @@ -4727,6 +4745,7 @@ export function heartbeatService( reconcileCostAccounting, buildRunOutputSilence, + buildRunOutputSilences, tickTimers, tickAgentTimers, diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 591bb30dd57..568e46ff21c 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -2432,6 +2432,24 @@ export function recoveryService( return watchdog.buildRunOutputSilence(run, now); } + async function buildRunOutputSilences( + runs: ReadonlyArray>, + now = new Date(), + ): Promise { + return watchdog.buildRunOutputSilences(runs, now); + } + async function appendRecoveryRunEvent( run: typeof heartbeatRuns.$inferSelect, event: { @@ -6845,6 +6863,7 @@ export function recoveryService( return { assignedWork, buildRunOutputSilence, + buildRunOutputSilences, escalateStrandedRecoveryIssueInPlace, escalateStrandedAssignedIssue, recordWatchdogDecision,