diff --git a/doc/mcp-discovery-performance.md b/doc/mcp-discovery-performance.md index 219079cf3e5..3ec87f17a4c 100644 --- a/doc/mcp-discovery-performance.md +++ b/doc/mcp-discovery-performance.md @@ -25,6 +25,41 @@ The stateless gateway and runtime-tools MCP endpoints return HTTP 405 with The runtime-tools endpoint still validates its caller before that response. See the [MCP transport contract](https://modelcontextprotocol.io/specification/2025-03-26/basic/transports). +## New-tool review projection + +Pending-tool review builds covered application and connection sets from each +profile's include rules. An included catalog entry covers its connection and its +non-null application only while that entry exists in the supplied catalog. +Explicit application selectors retain null application matching. A second index +records all exact selectors, including exclusions, so already reviewed tools do +not appear as new. The catalog filter retains its order, status check, strict +review watermark, and returned metadata. + +The indexes live only for one projection. Profile listings, new-tool review, and +app attention still read current company-scoped database state on each request. +This replaces nested catalog-by-rule scans with linear indexing and filtering. +It changes neither database query counts nor response contracts. + +The PostgreSQL reproduction uses 200 selected profiles, 1,900 active tools, and +76,000 rules. Profiles alternate between 65 and 695 rules. Half reference 695 +removed catalog entries, while half select current tools. This exercises both +scope checks and explicit-review checks. It compares complete ordered results +from the real profile listing and app-attention services before timing them. + +Save baseline `profiles/projection.ts`, `profiles/service.ts`, and +`catalog/attention.ts` beside their current versions as `.baseline-projection.ts`, +`.baseline-service.ts`, and `.baseline-attention.ts`. Change the two baseline +consumers' projection imports to `.baseline-projection.js` with the same relative +path. Then run from `server/`: + +```sh +node --expose-gc --import tsx scripts/benchmark-profile-review-projection.ts --baseline-profiles "$PWD/src/services/tool-access/profiles/.baseline-service.ts" --baseline-attention "$PWD/src/services/tool-access/catalog/.baseline-attention.ts" +``` + +Remove the three baseline files after the comparison. The probe alternates five +samples per implementation and reports process CPU, elapsed time, actual query +count, and returned rows. No provider calls or hosted traffic are generated. + ## Reproduce the measurements Build workspace dependencies, then run from `server/`: diff --git a/server/scripts/benchmark-profile-review-projection.ts b/server/scripts/benchmark-profile-review-projection.ts new file mode 100644 index 00000000000..694f3a25704 --- /dev/null +++ b/server/scripts/benchmark-profile-review-projection.ts @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import { isDeepStrictEqual } from "node:util"; +import { pathToFileURL } from "node:url"; +import { createDb, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db"; +import { createToolProfileService } from "../src/services/tool-access/profiles/service.js"; +import { createToolAppAttention } from "../src/services/tool-access/catalog/attention.js"; +import { createProfileProjectionFixture } from "../src/__tests__/helpers/tool-profile-projection-fixture.js"; +const option = (name: string, fallback: string) => { const i = process.argv.indexOf(name); return i < 0 ? fallback : process.argv[i + 1] ?? fallback; }; +const baselineProfiles = option("--baseline-profiles", ""), baselineAttention = option("--baseline-attention", ""); +if (!baselineProfiles || !baselineAttention) throw new Error("--baseline-profiles and --baseline-attention modules required"); +const beforeProfiles = (await import(pathToFileURL(baselineProfiles).href)).createToolProfileService as typeof createToolProfileService; +const beforeAttention = (await import(pathToFileURL(baselineAttention).href)).createToolAppAttention as typeof createToolAppAttention; +const profiles = Number(option("--profiles", "200")), repeats = Number(option("--repeat", "5")); +if (!Number.isInteger(profiles) || profiles < 2 || !Number.isInteger(repeats) || repeats < 1) throw new Error("Positive fixture counts required"); +const database = await startEmbeddedPostgresTestDatabase("profile-review-projection-"); +const db = createDb(database.connectionString, { maxConnections: 1 }); +let queries = 0, rows = 0; +const measuredDb = new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args), execute = selection.execute.bind(selection); + selection.execute = async () => { const result = await execute(); queries++; rows += result.length; return result; }; + return selection; + }) as typeof builder.from; + return builder; + }; +} }); +try { + const fixture = await createProfileProjectionFixture(db, { profiles }); + const now = () => new Date("2026-02-01T00:00:00Z"); + const implementations = { + before: { profiles: beforeProfiles(measuredDb, now), attention: beforeAttention(measuredDb, now) }, + after: { profiles: createToolProfileService(measuredDb, now), attention: createToolAppAttention(measuredDb, now) }, + }; + const invoke = async (label: keyof typeof implementations) => { + const service = implementations[label]; + return { profiles: await service.profiles.listProfiles(fixture.company.id), attention: await service.attention(fixture.company.id) }; + }; + const baseline = await invoke("before"), treatment = await invoke("after"); + assert.ok(isDeepStrictEqual(baseline, treatment), "Complete profile and attention results must match baseline"); + console.log(JSON.stringify({ fixture: { profiles, tools: 1900, meanRulesPerProfile: 380, ruleCounts: [65, 695], removedTools: 695, agents: 12, staleReferenceProfiles: Math.floor(profiles / 2) }, equality: true, profilePendingCounts: [...new Set(treatment.profiles.map(p => p.newToolsPendingCount))] })); + for (let round = 0; round < repeats; round++) { + for (const label of round % 2 ? ["after", "before"] as const : ["before", "after"] as const) { + global.gc?.(); queries = 0; rows = 0; + const start = performance.now(), cpu = process.cpuUsage(); + const result = await invoke(label); + const elapsedMs = performance.now() - start, used = process.cpuUsage(cpu); + assert.equal(result.profiles.length, profiles); + console.log(JSON.stringify({ round, label, elapsedMs, cpuMs: (used.user + used.system) / 1000, queries, rows, profiles, forcedGc: Boolean(global.gc) })); + } + } +} finally { await db.$client.end({ timeout: 1 }); await database.cleanup(); } diff --git a/server/src/__tests__/helpers/tool-profile-projection-fixture.ts b/server/src/__tests__/helpers/tool-profile-projection-fixture.ts new file mode 100644 index 00000000000..9a20ff0dd2f --- /dev/null +++ b/server/src/__tests__/helpers/tool-profile-projection-fixture.ts @@ -0,0 +1,29 @@ +import { randomUUID } from "node:crypto"; +import { agents, companies, toolApplications, toolCatalogEntries, toolConnections, toolProfileEntries, toolProfiles, type Db } from "@paperclipai/db"; + +export async function createProfileProjectionFixture(db: Db, options: { profiles?: number; tools?: number; rules?: number } = {}) { + const profileCount = options.profiles ?? 200, toolCount = options.tools ?? 1900, ruleCount = options.rules ?? 380, maxRuleCount = options.rules ?? 695; + const [company] = await db.insert(companies).values({ name: "Profile projection fixture", issuePrefix: `PP${randomUUID().slice(0, 6)}` }).returning(); + await db.insert(agents).values(Array.from({ length: 12 }, (_, i) => ({ companyId: company.id, name: `Agent ${i}` }))); + const [application] = await db.insert(toolApplications).values({ companyId: company.id, name: "Projection application", type: "mcp_http" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId: company.id, applicationId: application.id, name: "Projection connection", uid: randomUUID(), transport: "mcp_remote", status: "active", enabled: true, healthStatus: "ok" }).returning(); + const catalog = await db.insert(toolCatalogEntries).values(Array.from({ length: toolCount + maxRuleCount }, (_, i) => ({ + companyId: company.id, applicationId: application.id, connectionId: connection.id, + name: `tool_${i}`, toolName: `tool_${i}`, versionHash: `v${i}`, riskLevel: "read" as const, + status: i < toolCount ? "active" as const : "removed" as const, + firstSeenAt: new Date(i < toolCount ? "2026-01-03T00:00:00Z" : "2026-01-01T00:00:00Z"), + }))).returning(); + const profiles = await db.insert(toolProfiles).values(Array.from({ length: profileCount }, (_, i) => ({ + companyId: company.id, profileKey: `selected-${i}`, name: `Selected profile ${i}`, defaultAction: "deny" as const, + createdAt: new Date("2026-01-02T00:00:00Z"), + }))).returning(); + for (let i = 0; i < profiles.length; i++) { + const count = i % 4 < 2 ? maxRuleCount : 2 * ruleCount - maxRuleCount; + const rules = (i % 2 ? catalog.slice(toolCount, toolCount + count) : catalog.slice(0, count)).map(entry => ({ + companyId: company.id, profileId: profiles[i]!.id, selectorType: "catalog_entry" as const, + effect: "include" as const, catalogEntryId: entry.id, + })); + for (let offset = 0; offset < rules.length; offset += 500) await db.insert(toolProfileEntries).values(rules.slice(offset, offset + 500)); + } + return { company, application, connection, catalog: catalog.slice(0, toolCount), profiles }; +} diff --git a/server/src/__tests__/tool-profile-pending-projection.test.ts b/server/src/__tests__/tool-profile-pending-projection.test.ts new file mode 100644 index 00000000000..6b2764acc44 --- /dev/null +++ b/server/src/__tests__/tool-profile-pending-projection.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { pendingNewToolsForProfile, profileCoversCatalogScope, profileEntryMatchesCatalog, type ProfileCatalogEntry } from "../services/tool-access/profiles/projection.js"; +import type { toolProfiles, toolProfileEntries } from "@paperclipai/db"; + +type Input = Parameters[0]; +const date = new Date("2026-01-01T00:00:00Z"); +const later = new Date("2026-01-02T00:00:00Z"); +function profile(overrides: Partial = {}): typeof toolProfiles.$inferSelect { + return { id: "profile", companyId: "company", profileKey: "selected", name: "Selected", description: null, status: "active", defaultAction: "deny", newToolsReviewedAt: null, metadata: {}, createdAt: date, updatedAt: date, ...overrides }; +} +function rule(overrides: Partial): typeof toolProfileEntries.$inferSelect { + return { id: "rule", companyId: "company", profileId: "profile", selectorType: "catalog_entry", effect: "include", applicationId: null, connectionId: null, catalogEntryId: null, toolName: null, riskLevel: null, conditions: null, createdAt: date, updatedAt: date, ...overrides }; +} +function legacy(input: Input) { + if (input.profile.status !== "active" || input.profile.defaultAction !== "deny") return []; + const catalogById = new Map(input.catalog.map(entry => [entry.id, entry])); + return input.catalog.filter(entry => (entry.status === "active" || entry.status === "quarantined") + && entry.firstSeenAt > (input.profile.newToolsReviewedAt ?? input.profile.createdAt) + && input.entries.some(rule => profileCoversCatalogScope({ entry: rule, catalogEntry: entry, catalogById })) + && !input.entries.some(rule => profileEntryMatchesCatalog(rule, entry))).map(entry => ({ + catalogEntryId: entry.id, applicationId: entry.applicationId, + applicationName: entry.applicationId ? input.applicationsById?.get(entry.applicationId)?.name ?? null : null, + connectionId: entry.connectionId, connectionName: input.connectionsById?.get(entry.connectionId)?.name ?? null, + toolName: entry.toolName, title: entry.title ?? null, description: entry.description ?? null, + capability: entry.riskLevel, riskLevel: entry.riskLevel, addedAt: entry.firstSeenAt, firstSeenAt: entry.firstSeenAt, + })); +} +const catalog: ProfileCatalogEntry[] = Array.from({ length: 48 }, (_, i): ProfileCatalogEntry => ({ + id: `tool-${i}`, applicationId: i % 3 ? `app-${i % 4}` : null, + connectionId: `connection-${i % 6}`, toolName: i % 7 ? `name-${i}` : "quoted' path\\tool_🙂", + riskLevel: i % 2 ? "read" : "write", status: i % 5 ? "active" : "quarantined", + firstSeenAt: i % 11 ? later : date, +})); + +describe("pending profile projection", () => { + it("matches the previous ordered projection for every selector, effect, watermark and nullable scope", () => { + const rules = catalog.flatMap(entry => (["include", "exclude"] as const).flatMap(effect => [ + rule({ effect, selectorType: "application", applicationId: entry.applicationId }), + rule({ effect, selectorType: "connection", connectionId: entry.connectionId }), + rule({ effect, selectorType: "catalog_entry", catalogEntryId: entry.id }), + rule({ effect, selectorType: "tool_name", toolName: entry.toolName }), + rule({ effect, selectorType: "risk_level", riskLevel: entry.riskLevel }), + ])); + const scopes = [rule({ selectorType: "application", applicationId: null }), rule({ selectorType: "connection", connectionId: "connection-1" }), rule({ catalogEntryId: "tool-2" }), rule({ catalogEntryId: "missing" })]; + for (const scope of scopes) for (const selected of [[], ...rules.map(entry => [entry]), rules.slice(0, 50), rules]) { + const input = { profile: profile(), catalog, entries: [scope, ...selected], applicationsById: new Map([["app-1", { id: "app-1", name: "Application" }]]), connectionsById: new Map([["connection-1", { id: "connection-1", name: "Connection" }]]) }; + expect(pendingNewToolsForProfile(input)).toEqual(legacy(input)); + } + for (const changes of [{ status: "archived" as const }, { defaultAction: "allow" as const }, { newToolsReviewedAt: later }]) { + const input = { profile: profile(changes), catalog, entries: scopes }; + expect(pendingNewToolsForProfile(input)).toEqual(legacy(input)); + } + }); + + it("does not broaden an individual tool's null application scope to unrelated null applications", () => { + const input = { profile: profile(), entries: [rule({ catalogEntryId: "tool-0" })], catalog }; + const ids = pendingNewToolsForProfile(input).map(entry => entry.catalogEntryId); + expect(ids).toContain("tool-6"); + expect(ids).not.toContain("tool-3"); + expect(pendingNewToolsForProfile(input)).toEqual(legacy(input)); + }); + + it("recomputes catalog references, exclusions and scope after the input changes", () => { + const input = { profile: profile(), entries: [rule({ catalogEntryId: "tool-2" })], catalog }; + expect(pendingNewToolsForProfile(input).length).toBeGreaterThan(0); + expect(pendingNewToolsForProfile({ ...input, catalog: catalog.filter(entry => entry.id !== "tool-2") })).toEqual([]); + expect(pendingNewToolsForProfile({ ...input, entries: [rule({ catalogEntryId: "tool-2", effect: "exclude" })] })).toEqual([]); + }); +}); diff --git a/server/src/__tests__/tool-profile-review-service.test.ts b/server/src/__tests__/tool-profile-review-service.test.ts new file mode 100644 index 00000000000..edfa2a4d33c --- /dev/null +++ b/server/src/__tests__/tool-profile-review-service.test.ts @@ -0,0 +1,36 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { createDb, toolCatalogEntries, toolProfileEntries, toolProfiles } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createProfileProjectionFixture } from "./helpers/tool-profile-projection-fixture.js"; +import { createToolProfileService } from "../services/tool-access/profiles/service.js"; +import { createToolAppAttention } from "../services/tool-access/catalog/attention.js"; +const support = await getEmbeddedPostgresTestSupport(); +const describeDatabase = support.supported ? describe : describe.skip; +describeDatabase("pending profile tools through profile and attention services", () => { + let db: ReturnType, database: Awaited>; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("profile-review-service-"); db = createDb(database.connectionString); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + it("preserves fresh pending counts in both owning services after review and catalog changes", async () => { + const f = await createProfileProjectionFixture(db, { profiles: 2, tools: 8, rules: 4 }); + const now = () => new Date("2026-02-01T00:00:00Z"), service = createToolProfileService(db, now), attention = createToolAppAttention(db, now); + const assertCounts = async (expected: number) => { + const profiles = await service.listProfiles(f.company.id); + expect(profiles.find(p => p.id === f.profiles[0].id)?.newToolsPendingCount).toBe(expected); + expect(profiles.find(p => p.id === f.profiles[1].id)?.newToolsPendingCount).toBe(0); + expect((await service.listProfileNewTools(f.profiles[0].id, f.company.id)).pendingCount).toBe(expected); + expect((await attention(f.company.id)).totals.newToolsPendingReview).toBe(expected); + }; + await assertCounts(4); + const [exclude] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profiles[0].id, + selectorType: "tool_name", effect: "exclude", toolName: f.catalog[4].toolName }).returning(); + await assertCounts(3); + await db.update(toolCatalogEntries).set({ status: "removed" }).where(eq(toolCatalogEntries.id, f.catalog[5].id)); + await assertCounts(2); + await db.update(toolProfiles).set({ newToolsReviewedAt: now() }).where(eq(toolProfiles.id, f.profiles[0].id)); + await assertCounts(0); + const foreign = await createProfileProjectionFixture(db, { profiles: 2, tools: 5, rules: 2 }); + expect((await service.listProfiles(foreign.company.id)).map(p => p.companyId)).toEqual([foreign.company.id, foreign.company.id]); + expect((await db.select().from(toolProfileEntries).where(eq(toolProfileEntries.id, exclude.id)))[0].effect).toBe("exclude"); + }); +}); diff --git a/server/src/services/tool-access/profiles/projection.ts b/server/src/services/tool-access/profiles/projection.ts index 53b92ef182f..58f64d0f20e 100644 --- a/server/src/services/tool-access/profiles/projection.ts +++ b/server/src/services/tool-access/profiles/projection.ts @@ -103,6 +103,24 @@ export function profileEntryMatchesCatalog( return false; } +export function profileCatalogMatcher(entries: ProfileEntryMatch[]) { + const applications = new Set(); + const connections = new Set(); + const catalogIds = new Set(); + const toolNames = new Set(); + const risks = new Set(); + for (const entry of entries) { + if (entry.selectorType === "application") applications.add(entry.applicationId); + else if (entry.selectorType === "connection") connections.add(entry.connectionId); + else if (entry.selectorType === "catalog_entry") catalogIds.add(entry.catalogEntryId); + else if (entry.selectorType === "tool_name") toolNames.add(entry.toolName); + else if (entry.selectorType === "risk_level") risks.add(entry.riskLevel); + } + return (entry: ProfileCatalogMatch) => applications.has(entry.applicationId) + || connections.has(entry.connectionId) || catalogIds.has(entry.id) + || toolNames.has(entry.toolName) || risks.has(entry.riskLevel); +} + export function summarizeProfile(input: { profile: typeof toolProfiles.$inferSelect; entries: Array; @@ -203,32 +221,28 @@ export function pendingNewToolsForProfile(input: { return []; const watermark = input.profile.newToolsReviewedAt ?? input.profile.createdAt; const catalogById = new Map(input.catalog.map((entry) => [entry.id, entry])); - const scopedIncludes = input.entries.filter( - (entry) => - entry.effect === "include" && - (entry.selectorType === "application" || - entry.selectorType === "connection" || - entry.selectorType === "catalog_entry"), - ); - if (scopedIncludes.length === 0) return []; + const coveredApplications = new Set(); + const coveredConnections = new Set(); + for (const entry of input.entries) { + if (entry.effect !== "include") continue; + if (entry.selectorType === "application") coveredApplications.add(entry.applicationId); + else if (entry.selectorType === "connection") coveredConnections.add(entry.connectionId); + else if (entry.selectorType === "catalog_entry" && entry.catalogEntryId) { + const scopedEntry = catalogById.get(entry.catalogEntryId); + if (!scopedEntry) continue; + coveredConnections.add(scopedEntry.connectionId); + if (scopedEntry.applicationId) coveredApplications.add(scopedEntry.applicationId); + } + } + if (coveredApplications.size === 0 && coveredConnections.size === 0) return []; + const explicitlyReviewed = profileCatalogMatcher(input.entries); return input.catalog - .filter( - (catalogEntry) => - catalogEntry.status === "active" || - catalogEntry.status === "quarantined", - ) - .filter((catalogEntry) => catalogEntry.firstSeenAt > watermark) .filter((catalogEntry) => - scopedIncludes.some((entry) => - profileCoversCatalogScope({ entry, catalogEntry, catalogById }), - ), - ) - .filter( - (catalogEntry) => - !input.entries.some((entry) => - profileEntryMatchesCatalog(entry, catalogEntry), - ), + (catalogEntry.status === "active" || catalogEntry.status === "quarantined") + && catalogEntry.firstSeenAt > watermark + && (coveredApplications.has(catalogEntry.applicationId) || coveredConnections.has(catalogEntry.connectionId)) + && !explicitlyReviewed(catalogEntry), ) .map((catalogEntry) => ({ catalogEntryId: catalogEntry.id,