diff --git a/doc/mcp-discovery-performance.md b/doc/mcp-discovery-performance.md index 3ec87f17a4c..ac696b01187 100644 --- a/doc/mcp-discovery-performance.md +++ b/doc/mcp-discovery-performance.md @@ -106,3 +106,35 @@ Catalog payload memory and policy-matching CPU still grow with the catalog. The measurements do not establish a safe production heap limit or the share of active installations affected. Verify restart bursts and sustained traffic on the actual deployment before reducing its temporary heap allowance. + +## Profile selector matching + +Effective profile resolution and profile summaries index each profile's include +and exclude selectors once per decision. Catalog rows then use exact set +membership checks for application, connection, catalog entry, tool name, and +risk level. Exclusions still take precedence within a profile; effective profiles +still contribute the union of their allowed tools. Null application selectors +retain their exact-match behavior. + +These indexes live only for the current computation. Each resolver call still +reads current company-scoped bindings, profiles, and rules. Query counts and +returned database rows are unchanged. This reduces JavaScript matching work when +a profile selects many individual tools; it does not cache authorization or +remove the runtime assignment digest check. + +For a real PostgreSQL comparison, save `projection.ts` and `resolution.ts` from +the baseline commit beside their current versions as `.baseline-projection.ts` +and `.baseline-resolution.ts`. Change only the baseline resolver's projection +import to `./.baseline-projection.js`. Then run from `server/`: + +```sh +node --import tsx scripts/benchmark-tool-profile-matching.ts --baseline "$PWD/src/services/tool-access/profiles/.baseline-resolution.ts" +``` + +Remove the two baseline files afterward. The probe uses 1,890 catalog tools, +1,575 individual include/exclude rules, one effective profile, and 12 agents. It +checks full assignment-result equality, warms both versions, and alternates +seven samples of ten resolutions. It reports actual query and returned-row +counts, process CPU, and elapsed time. The installation dependency returns an +empty list for this fixture, which has no installed connections. It does not +measure provider work, whole-run latency, or hosted performance. diff --git a/server/scripts/benchmark-tool-profile-matching.ts b/server/scripts/benchmark-tool-profile-matching.ts new file mode 100644 index 00000000000..f0e0e28ce73 --- /dev/null +++ b/server/scripts/benchmark-tool-profile-matching.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import { pathToFileURL } from "node:url"; +import { and, eq } from "drizzle-orm"; +import { agents, createDb, type Db } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "../src/__tests__/helpers/embedded-postgres.js"; +import { createToolProfileMatchingFixture } from "../src/__tests__/helpers/tool-profile-matching-fixture.js"; +import { createToolProfileResolver } from "../src/services/tool-access/profiles/resolution.js"; + +const baselinePath = process.argv[process.argv.indexOf("--baseline") + 1]; +if (!process.argv.includes("--baseline") || !baselinePath) throw new Error("--baseline module required"); +const baselineFactory = (await import(pathToFileURL(baselinePath).href)).createToolProfileResolver as typeof createToolProfileResolver; +const database = await startEmbeddedPostgresTestDatabase("profile-matching-benchmark-"); +const db = createDb(database.connectionString, { maxConnections: 1 }); +let queries = 0, rows = 0; +const observedDb = new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args), execute = selection.execute.bind(selection); + selection.execute = async () => { const result = await execute(); queries++; rows += result.length; return result; }; + return selection; + }) as typeof builder.from; + return builder; + }; +} }); +try { + const { company, agent } = await createToolProfileMatchingFixture(db); + const dependencies = { + assertOptionalAgent: async (companyId: string, agentId: string | null | undefined) => { + const valid = await observedDb.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, companyId), eq(agents.id, agentId!))); + assert.equal(valid.length, 1); + }, + resolveInstalledConnectionsForAgent: async () => [], + toCatalogEntry: () => { throw new Error("Full catalog definitions are not used by this assignment benchmark"); }, + }; + const before = baselineFactory(observedDb, dependencies); + const after = createToolProfileResolver(observedDb, dependencies); + const invoke = (resolver: typeof after) => resolver.getEffectiveToolAssignmentsForAgent(company.id, agent.id); + assert.deepEqual(await invoke(after), await invoke(before)); + for (let i = 0; i < 2; i++) { await invoke(before); await invoke(after); } + for (let round = 0; round < 7; round++) { + for (const [label, resolver] of round % 2 ? [["after", after], ["before", before]] as const : [["before", before], ["after", after]] as const) { + queries = 0; rows = 0; + const cpu = process.cpuUsage(), start = performance.now(); + for (let i = 0; i < 10; i++) await invoke(resolver); + const elapsedMs = performance.now() - start, used = process.cpuUsage(cpu); + console.log(JSON.stringify({ round, label, operations: 10, tools: 1890, rules: 1575, agents: 12, queries, rows, elapsedMs, cpuMs: (used.user + used.system) / 1000 })); + } + } +} finally { await db.$client.end({ timeout: 1 }); await database.cleanup(); } diff --git a/server/src/__tests__/helpers/tool-profile-matching-fixture.ts b/server/src/__tests__/helpers/tool-profile-matching-fixture.ts new file mode 100644 index 00000000000..087e7c8bcb9 --- /dev/null +++ b/server/src/__tests__/helpers/tool-profile-matching-fixture.ts @@ -0,0 +1,15 @@ +import { randomUUID } from "node:crypto"; +import { agents, companies, toolApplications, toolCatalogEntries, toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db } from "@paperclipai/db"; + +export async function createToolProfileMatchingFixture(db: Db) { + const [company] = await db.insert(companies).values({ name: "Synthetic profile matching", issuePrefix: `BM${randomUUID().slice(0, 6)}` }).returning(); + const companyAgents = await db.insert(agents).values(Array.from({ length: 12 }, (_, i) => ({ companyId: company.id, name: `Agent ${i}` }))).returning(); + const agent = companyAgents[0]!; + const [app] = await db.insert(toolApplications).values({ companyId: company.id, name: "Synthetic app", type: "mcp_http" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId: company.id, applicationId: app.id, name: "Synthetic connection", uid: randomUUID(), transport: "mcp_remote" }).returning(); + const catalog = await db.insert(toolCatalogEntries).values(Array.from({ length: 1890 }, (_, i) => ({ companyId: company.id, applicationId: app.id, connectionId: connection.id, name: `tool_${i}`, toolName: `tool_${i}`, versionHash: `v${i}`, riskLevel: "read" as const }))).returning(); + const [profile] = await db.insert(toolProfiles).values({ companyId: company.id, profileKey: "selected", name: "Selected tools", defaultAction: "deny" }).returning(); + await db.insert(toolProfileBindings).values({ companyId: company.id, profileId: profile.id, targetType: "agent", targetId: agent.id }); + await db.insert(toolProfileEntries).values(catalog.slice(0, 1575).map((entry, i) => ({ companyId: company.id, profileId: profile.id, selectorType: "catalog_entry" as const, catalogEntryId: entry.id, effect: i % 5 === 0 ? "exclude" as const : "include" as const }))); + return { company, agent }; +} diff --git a/server/src/__tests__/tool-effective-assignments.test.ts b/server/src/__tests__/tool-effective-assignments.test.ts index 88534789025..f22b8fd0cb6 100644 --- a/server/src/__tests__/tool-effective-assignments.test.ts +++ b/server/src/__tests__/tool-effective-assignments.test.ts @@ -76,6 +76,20 @@ describeDatabase("effective tool assignments", () => { expect(full.allowedTools[0].annotations).toEqual(f.read.annotations); }); + it("keeps exclusions local to each profile when effective profiles grant the same tool", async () => { + const f = await fixture(); + await db.update(toolProfiles).set({ defaultAction: "allow" }).where(eq(toolProfiles.id, f.profile.id)); + await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profile.id, + selectorType: "catalog_entry", effect: "exclude", catalogEntryId: f.read.id }); + const [second] = await db.insert(toolProfiles).values({ companyId: f.company.id, profileKey: "second", name: "Second", defaultAction: "deny" }).returning(); + await db.insert(toolProfileBindings).values({ companyId: f.company.id, profileId: second.id, targetType: "agent", targetId: f.agent.id }); + const [include] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: second.id, + selectorType: "catalog_entry", effect: "include", catalogEntryId: f.read.id }).returning(); + expect((await compare(f)).assignments.allowedTools.map(tool => tool.id)).toEqual([f.read.id, f.write.id]); + await db.delete(toolProfileEntries).where(eq(toolProfileEntries.id, include.id)); + expect((await compare(f)).assignments.allowedTools.map(tool => tool.id)).toEqual([f.write.id]); + }); + it("keeps unresolved exact names and observes changed catalog, profile and binding state", async () => { const f = await fixture(); await db.insert(toolProfileEntries).values([ diff --git a/server/src/__tests__/tool-profile-matching.test.ts b/server/src/__tests__/tool-profile-matching.test.ts new file mode 100644 index 00000000000..4c8498ed6bd --- /dev/null +++ b/server/src/__tests__/tool-profile-matching.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { profileCatalogMatcher, profileEntryMatchesCatalog, type ProfileCatalogMatch, type ProfileEntryMatch } from "../services/tool-access/profiles/projection.js"; + +describe("indexed profile catalog matching", () => { + it("preserves all selector kinds, null matches and exact strings in mixed rule sets", () => { + const catalog: ProfileCatalogMatch[] = Array.from({ length: 75 }, (_, index) => ({ + id: `tool-${index}`, applicationId: index % 3 ? `app-${index % 5}` : null, + connectionId: `connection-${index % 7}`, toolName: index % 4 ? `tool-${index}` : "quoted' path\\tool_🙂", + riskLevel: index % 2 ? "read" : "write", + })); + const selectors = ["application", "connection", "catalog_entry", "tool_name", "risk_level", "unknown"]; + const rules: ProfileEntryMatch[] = catalog.flatMap(entry => selectors.map(selectorType => ({ + selectorType, applicationId: entry.applicationId, connectionId: entry.connectionId, + catalogEntryId: entry.id, toolName: entry.toolName, riskLevel: entry.riskLevel, + }))); + for (const selected of [[], ...rules.map(rule => [rule]), rules.slice(0, 25), rules]) { + const matches = profileCatalogMatcher(selected); + expect(catalog.filter(matches)).toEqual(catalog.filter(entry => selected.some(rule => profileEntryMatchesCatalog(rule, entry)))); + } + }); +}); diff --git a/server/src/services/tool-access/profiles/projection.ts b/server/src/services/tool-access/profiles/projection.ts index 58f64d0f20e..877c2606fd4 100644 --- a/server/src/services/tool-access/profiles/projection.ts +++ b/server/src/services/tool-access/profiles/projection.ts @@ -128,22 +128,18 @@ export function summarizeProfile(input: { catalog: ProfileCatalogEntry[]; agentIds: string[]; }): ToolProfileSummary { - const includes = input.entries.filter((entry) => entry.effect === "include"); - const excludes = input.entries.filter((entry) => entry.effect === "exclude"); + const included = profileCatalogMatcher(input.entries.filter((entry) => entry.effect === "include")); + const excluded = profileCatalogMatcher(input.entries.filter((entry) => entry.effect === "exclude")); const allowedCatalogIds = new Set(); const allowedApplicationIds = new Set(); const excludedCatalogIds = new Set(); for (const catalogEntry of input.catalog) { - const excluded = excludes.some((entry) => - profileEntryMatchesCatalog(entry, catalogEntry), - ); - if (excluded) excludedCatalogIds.add(catalogEntry.id); - if (excluded) continue; - const included = includes.some((entry) => - profileEntryMatchesCatalog(entry, catalogEntry), - ); - if (input.profile.defaultAction === "allow" || included) { + if (excluded(catalogEntry)) { + excludedCatalogIds.add(catalogEntry.id); + continue; + } + if (input.profile.defaultAction === "allow" || included(catalogEntry)) { allowedCatalogIds.add(catalogEntry.id); if (catalogEntry.applicationId) allowedApplicationIds.add(catalogEntry.applicationId); diff --git a/server/src/services/tool-access/profiles/resolution.ts b/server/src/services/tool-access/profiles/resolution.ts index c237636eb2b..b06683b8203 100644 --- a/server/src/services/tool-access/profiles/resolution.ts +++ b/server/src/services/tool-access/profiles/resolution.ts @@ -3,7 +3,7 @@ import type { Db } from "@paperclipai/db"; import { agents, toolCatalogEntries, toolProfileBindings, toolProfileEntries, toolProfiles } from "@paperclipai/db"; import type { ToolCatalogEntry, ToolConnection, ToolProfileEffectiveSummary, ToolProfileWithDetails } from "@paperclipai/shared"; import { effectiveToolProfileBindings, profileIdsInBindingOrder } from "../../tool-profile-binding-precedence.js"; -import { buildProfileDetails, profileCatalogColumns, profileEntryMatchesCatalog, toProfileBinding, toProfileEntry, type ProfileCatalogEntry, type ProfileEntryMatch, type ProfileCatalogMatch } from "./projection.js"; +import { buildProfileDetails, profileCatalogColumns, profileCatalogMatcher, toProfileBinding, toProfileEntry, type ProfileCatalogEntry, type ProfileEntryMatch, type ProfileCatalogMatch } from "./projection.js"; type ProfileAssignmentDependencies = { assertOptionalAgent(companyId: string, agentId: string | null | undefined, label: string): Promise; resolveInstalledConnectionsForAgent(companyId: string, agentId: string): Promise; @@ -37,18 +37,13 @@ function resolveCatalogAccess( const excludes = profileEntries.filter( (entry) => entry.effect === "exclude", ); + const included = profileCatalogMatcher(includes); + const excluded = profileCatalogMatcher(excludes); for (const catalogEntry of catalog) { - if ( - excludes.some((entry) => - profileEntryMatchesCatalog(entry, catalogEntry), - ) - ) - continue; + if (excluded(catalogEntry)) continue; if ( profile.defaultAction === "allow" || - includes.some((entry) => - profileEntryMatchesCatalog(entry, catalogEntry), - ) + included(catalogEntry) ) { allowedCatalogIds.add(catalogEntry.id); allowedToolNames.add(catalogEntry.toolName);