diff --git a/doc/DATABASE.md b/doc/DATABASE.md index c4d5c72e41e..6c47e5d7495 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -231,6 +231,13 @@ The benchmark loads the baseline authorization module with current dependencies, compares complete workspace rows, and prints five warm samples and query plans. It measures SQL execution, not the full repository admission or agent startup. +Agent lists read the complete company roster once and use that same snapshot for +termination filtering and organization-chain normalization. Detail and plugin runtime reads +load only identity, company, name, status, and manager from other company agents +for chain checks. The selected agent's full configuration, current monthly spend, +and existing response redaction remain available. Every call reads current data; +these lookups do not cache authorization or agent configuration. + ## 1. Embedded PostgreSQL — zero config If you don't set `DATABASE_URL`, the server automatically starts an embedded PostgreSQL instance and manages a local data directory. diff --git a/server/src/__tests__/agent-record-read-projections.test.ts b/server/src/__tests__/agent-record-read-projections.test.ts new file mode 100644 index 00000000000..bf0a5183202 --- /dev/null +++ b/server/src/__tests__/agent-record-read-projections.test.ts @@ -0,0 +1,132 @@ +import { randomUUID } from "node:crypto"; +import { agents, companies, costEvents, createDb } from "@paperclipai/db"; +import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { agentRecordQueries } from "../lib/agent-records.js"; +import { agentService } from "../services/agents.js"; + +const support = await getEmbeddedPostgresTestSupport(); + +describe.skipIf(!support.supported)("agent roster read projections", () => { + let database: Awaited>; + let db: ReturnType; + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("agent-roster-read-"); + db = createDb(database.connectionString); + }, 60_000); + + beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2020-06-15T12:00:00Z")); + }); + + afterEach(() => { vi.useRealTimers(); }); + + afterAll(async () => { + await db?.$client.end(); + await database?.cleanup(); + }); + + async function fixture() { + const companyId = randomUUID(), foreignCompanyId = randomUUID(); + const managerId = randomUUID(), workerId = randomUUID(), unrelatedId = randomUUID(), archivedId = randomUUID(), foreignId = randomUUID(); + await db.insert(companies).values([ + { id: companyId, name: "Roster fixture", issuePrefix: `R${companyId.slice(0, 7).toUpperCase()}` }, + { id: foreignCompanyId, name: "Foreign roster", issuePrefix: `F${foreignCompanyId.slice(0, 7).toUpperCase()}` }, + ]); + const bulk = "retained-other-agent-configuration🙂".repeat(32_768); + await db.insert(agents).values([ + { id: managerId, companyId, name: "Manager", status: "terminated", adapterType: "process", adapterConfig: { instructions: bulk } }, + { id: unrelatedId, companyId, name: "Other worker", status: "idle", adapterType: "process", adapterConfig: { instructions: bulk } }, + { id: archivedId, companyId, name: "Archived worker", status: "terminated", adapterType: "process", runtimeConfig: { retained: bulk } }, + { id: foreignId, companyId: foreignCompanyId, name: "Foreign manager", status: "idle", adapterType: "process", adapterConfig: { instructions: bulk } }, + ]); + const [worker] = await db.insert(agents).values({ id: workerId, companyId, name: "Selected worker", status: "idle", + reportsTo: managerId, adapterType: "process", adapterConfig: { instructions: "selected instructions", model: "fixture-model" }, + runtimeConfig: { heartbeat: { enabled: false } }, metadata: { retained: "selected metadata" }, + permissions: { canCreateAgents: true }, spentMonthlyCents: 999, + }).returning(); + const now = new Date(); + await db.insert(costEvents).values([ + { companyId, agentId: workerId, provider: "fixture", model: "fixture-model", costCents: 19.5, occurredAt: now }, + { companyId, agentId: workerId, provider: "fixture", model: "fixture-model", costCents: 50, + occurredAt: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 1, 15)) }, + { companyId: foreignCompanyId, agentId: foreignId, provider: "fixture", model: "fixture-model", costCents: 100, occurredAt: now }, + ]); + return { companyId, foreignCompanyId, managerId, workerId, unrelatedId, archivedId, foreignId, worker, bulk }; + } + + async function observe(run: (observed: ReturnType) => Promise) { + const observed = createDb(database.connectionString), events: Array> = [], now = Date.now(); + const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + installDatabaseResponseCapture(observed, { agents, costEvents }, capture); + try { + const result = await run(observed); + capture.flush(); + const reads = events.filter(event => event.table === "agents" && event.operation === "select"); + return { result, calls: reads.reduce((n, event) => n + Number(event.calls), 0), + bytes: reads.reduce((n, event) => n + Number(event.responseValueBytes), 0), events }; + } finally { + capture.flush(); + await observed.$client.end(); + } + } + + it.each([false, true])("reads one full roster for list, preserving terminated-manager evidence and configurations (includeTerminated=%s)", async includeTerminated => { + const f = await fixture(); + const captured = await observe(observed => agentService(observed).list(f.companyId, { includeTerminated })); + expect(captured.calls).toBe(1); + expect(captured.result.map(row => row.id).sort()).toEqual((includeTerminated + ? [f.workerId, f.managerId, f.archivedId, f.unrelatedId] : [f.workerId, f.unrelatedId]).sort()); + const selected = captured.result.find(row => row.id === f.workerId)!; + expect(selected).toMatchObject({ adapterConfig: f.worker.adapterConfig, runtimeConfig: f.worker.runtimeConfig, + metadata: f.worker.metadata, permissions: { canCreateAgents: true }, spentMonthlyCents: 19.5, + orgChainHealth: { status: "invalid_org_chain", reason: "terminated_ancestor", firstInvalidAncestor: { id: f.managerId } } }); + expect(captured.result.find(row => row.id === f.unrelatedId)?.adapterConfig.instructions).toBe(f.bulk); + if (includeTerminated) expect(captured.result.find(row => row.id === f.archivedId)?.runtimeConfig.retained).toBe(f.bulk); + expect((await db.select().from(agents).where(eq(agents.id, f.workerId)))[0]).toEqual(f.worker); + expect(JSON.stringify(captured.events)).not.toContain("retained-other-agent-configuration"); + }); + + it("keeps detail reads small while returning the selected agent's complete current configuration", async () => { + const f = await fixture(); + const captured = await observe(observed => agentRecordQueries(observed).getById(f.workerId)); + expect(captured.calls).toBe(2); + expect(captured.bytes).toBeLessThan(8192); + expect(captured.result).toMatchObject({ id: f.workerId, companyId: f.companyId, adapterConfig: f.worker.adapterConfig, + runtimeConfig: f.worker.runtimeConfig, metadata: f.worker.metadata, permissions: { canCreateAgents: true }, spentMonthlyCents: 19.5, + orgChainHealth: { reason: "terminated_ancestor", firstInvalidAncestor: { id: f.managerId, name: "Manager" } } }); + expect((await db.select().from(agents).where(eq(agents.id, f.workerId)))[0]).toEqual(f.worker); + }); + + it("projects only current company chain fields for plugin normalization", async () => { + const f = await fixture(); + const captured = await observe(observed => agentRecordQueries(observed).listCompanyAgentEligibilityRows(f.companyId)); + expect(captured.calls).toBe(1); + expect(captured.bytes).toBeLessThan(4096); + expect(captured.result.map(row => row.id).sort()).toEqual([f.workerId, f.managerId, f.unrelatedId, f.archivedId].sort()); + for (const row of captured.result) expect(Object.keys(row).sort()).toEqual(["companyId", "id", "name", "reportsTo", "status"]); + const normalized = agentRecordQueries(db).normalizeAgentRow(f.worker, captured.result); + expect(normalized).toMatchObject({ adapterConfig: f.worker.adapterConfig, metadata: f.worker.metadata, + orgChainHealth: { reason: "terminated_ancestor", firstInvalidAncestor: { id: f.managerId } } }); + expect(await agentRecordQueries(db).listCompanyAgentEligibilityRows(randomUUID())).toEqual([]); + expect(await agentRecordQueries(db).getById(randomUUID())).toBeNull(); + }); + + it("rechecks manager status, name, selected configuration, missing managers, and cycles on later reads", async () => { + const f = await fixture(), records = agentRecordQueries(db); + expect((await records.getById(f.workerId))?.orgChainHealth.reason).toBe("terminated_ancestor"); + await db.update(agents).set({ status: "paused", name: "Current manager" }).where(eq(agents.id, f.managerId)); + await db.update(agents).set({ adapterConfig: { instructions: "changed instructions" } }).where(eq(agents.id, f.workerId)); + expect(await records.getById(f.workerId)).toMatchObject({ adapterConfig: { instructions: "changed instructions" }, + orgChainHealth: { status: "healthy", pausedAncestors: [{ id: f.managerId, name: "Current manager", status: "paused" }] } }); + await db.update(agents).set({ reportsTo: f.foreignId }).where(eq(agents.id, f.workerId)); + expect((await records.getById(f.workerId))?.orgChainHealth.reason).toBe("missing_manager"); + await db.update(agents).set({ reportsTo: f.managerId }).where(eq(agents.id, f.workerId)); + await db.update(agents).set({ reportsTo: f.workerId }).where(eq(agents.id, f.managerId)); + expect((await records.getById(f.workerId))?.orgChainHealth.reason).toBe("cycle"); + }); +}); diff --git a/server/src/lib/agent-records.ts b/server/src/lib/agent-records.ts index 786f61af550..25c0ed7c9f5 100644 --- a/server/src/lib/agent-records.ts +++ b/server/src/lib/agent-records.ts @@ -98,6 +98,8 @@ export interface CreateAgentOptions { claudeLogin?: ClaudeLoginContext; } +type AgentEligibilityRow = Pick; + interface AgentShortnameRow { id: string; name: string; @@ -340,7 +342,7 @@ export function agentRecordQueries(db: Db) { }; } - function normalizeAgentRows(rows: (typeof agents.$inferSelect)[], allCompanyRows = rows) { + function normalizeAgentRows(rows: (typeof agents.$inferSelect)[], allCompanyRows: AgentEligibilityRow[] = rows) { const eligibilityAgents = allCompanyRows.map(toEligibilityAgent); return rows.map((row) => { const base = normalizeAgentBaseRow(row); @@ -357,7 +359,7 @@ export function agentRecordQueries(db: Db) { }); } - function normalizeAgentRow(row: typeof agents.$inferSelect, allCompanyRows?: (typeof agents.$inferSelect)[]) { + function normalizeAgentRow(row: typeof agents.$inferSelect, allCompanyRows?: AgentEligibilityRow[]) { return normalizeAgentRows([row], allCompanyRows)[0]!; } @@ -365,6 +367,11 @@ export function agentRecordQueries(db: Db) { return db.select().from(agents).where(eq(agents.companyId, companyId)); } + async function listCompanyAgentEligibilityRows(companyId: string): Promise { + return db.select({ id: agents.id, companyId: agents.companyId, name: agents.name, + status: agents.status, reportsTo: agents.reportsTo }).from(agents).where(eq(agents.companyId, companyId)); + } + async function getMonthlySpendByAgentIds(companyId: string, agentIds: string[]) { if (agentIds.length === 0) return new Map(); const { start, end } = currentUtcMonthWindow(); @@ -405,7 +412,7 @@ export function agentRecordQueries(db: Db) { .then((rows) => rows[0] ?? null); if (!row) return null; const [companyRows, hydrated] = await Promise.all([ - listCompanyAgentRows(row.companyId), + listCompanyAgentEligibilityRows(row.companyId), hydrateAgentSpend([row]).then((rows) => rows[0]!), ]); return normalizeAgentRow(hydrated, companyRows); @@ -472,6 +479,6 @@ export function agentRecordQueries(db: Db) { }); } - return { normalizeAgentRows, normalizeAgentRow, listCompanyAgentRows, hydrateAgentSpend, + return { normalizeAgentRows, normalizeAgentRow, listCompanyAgentRows, listCompanyAgentEligibilityRows, hydrateAgentSpend, getById, ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed }; } diff --git a/server/src/services/agents.ts b/server/src/services/agents.ts index 4c4cbd7248f..c9dd824f495 100644 --- a/server/src/services/agents.ts +++ b/server/src/services/agents.ts @@ -1,6 +1,6 @@ import { type BudgetServiceHooks } from "./budgets.js"; -import { and, desc, eq, inArray, ne } from "drizzle-orm"; +import { and, desc, eq, inArray } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, agentConfigRevisions, agentApiKeys, heartbeatRuns } from "@paperclipai/db"; import { isUuidLike, normalizeAgentApiKeyScope, normalizeAgentUrlKey, type AgentApiKeyScope } from "@paperclipai/shared"; @@ -31,14 +31,10 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) { createAgentLifecycle(db, budgetHooks).updateConfiguration(...args); return { list: async (companyId: string, options?: { includeTerminated?: boolean }) => { - const conditions = [eq(agents.companyId, companyId)]; - if (!options?.includeTerminated) { - conditions.push(ne(agents.status, "terminated")); - } - const [rows, allCompanyRows] = await Promise.all([ - db.select().from(agents).where(and(...conditions)), - listCompanyAgentRows(companyId), - ]); + const allCompanyRows = await listCompanyAgentRows(companyId); + const rows = options?.includeTerminated + ? allCompanyRows + : allCompanyRows.filter((row) => row.status !== "terminated"); const hydrated = await hydrateAgentSpend(rows); return normalizeAgentRows(hydrated, allCompanyRows); }, diff --git a/server/src/services/plugin-agent-runtime-state.ts b/server/src/services/plugin-agent-runtime-state.ts index a1a4ce22ca7..0770246ef14 100644 --- a/server/src/services/plugin-agent-runtime-state.ts +++ b/server/src/services/plugin-agent-runtime-state.ts @@ -36,7 +36,7 @@ async function readState(db: Db, pluginId: string, companyId: string, agentId: s }; const instructions = await agentInstructionsService().getBundleLocation(agent); const records = agentRecordQueries(tx as unknown as Db); - const companyAgents = await records.listCompanyAgentRows(companyId); + const companyAgents = await records.listCompanyAgentEligibilityRows(companyId); return { revision: String(latest?.id ?? 0), agent: records.normalizeAgentRow(agent, companyAgents) as PluginAgentRuntimeState["agent"], instructionsRootPath: instructions.rootPath, instructionsEntryFile: instructions.entryFile,