diff --git a/doc/mcp-discovery-performance.md b/doc/mcp-discovery-performance.md index bed7a912efd..58d06222aee 100644 --- a/doc/mcp-discovery-performance.md +++ b/doc/mcp-discovery-performance.md @@ -139,6 +139,30 @@ counts, process CPU, and elapsed time. The installation dependency returns an empty list for this fixture, which has no installed connections. It does not measure provider work, whole-run latency, or hosted performance. +## Policy selector matching + +A listing decides access once per tool and shares one decision cache. The cache +holds the company's enabled policies compiled once: selectors are parsed once, +and policies are indexed by catalog entry, then by tool name for policies +without a catalog entry selector. Each decision checks only the policies in the +matching buckets plus the policies that select neither key, in the original +priority and creation order. Conditions are parsed once per policy. Decisions +without a cache, including those that consume a rate limit, compile the policies +they read in the same way. Decisions are unchanged. + +To compare against a baseline commit on real PostgreSQL, run from `server/`: + +```sh +pnpm exec tsx scripts/benchmark-tool-policy-matching.ts --baseline-ref origin/master --tools 800 --rounds 9 +``` + +The probe uses 800 catalog tools and 365 enabled policies shaped like the +staging company that showed this cost (342 select a catalog entry, 18 a catalog +entry and connection, 3 several other keys, 17 carry conditions). It checks that +both versions return the same decisions, then alternates one listing per version +per round and reports elapsed time, process CPU, the longest event-loop block, +and selector work per listing. + ## Local profile comparison on 2026-11-02 diff --git a/server/scripts/benchmark-tool-policy-matching.ts b/server/scripts/benchmark-tool-policy-matching.ts new file mode 100644 index 00000000000..6f9d4073999 --- /dev/null +++ b/server/scripts/benchmark-tool-policy-matching.ts @@ -0,0 +1,183 @@ +/** + * Main-thread cost of the policy decisions in one tools/list response, against + * enabled policies shaped like the staging company (see + * stagingShapedToolPolicies). Uses a throwaway embedded PostgreSQL database; no + * provider calls, production data, or model inference. Alternates the baseline + * and current policy services, one listing (fresh decision cache) per run. + * + * pnpm --filter @paperclipai/server exec tsx scripts/benchmark-tool-policy-matching.ts \ + * --baseline-ref origin/master --tools 800 --rounds 9 + */ +import { execFileSync } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { rmSync, writeFileSync } from "node:fs"; +import { setTimeout as sleep } from "node:timers/promises"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { monitorEventLoopDelay, performance } from "node:perf_hooks"; +import { createDb, toolPolicies, type Db as DbClient } from "@paperclipai/db"; +import type { ToolAccessDecisionInput } from "@paperclipai/shared"; +import { startEmbeddedPostgresTestDatabase } from "../src/__tests__/helpers/embedded-postgres.js"; +import { createListingFixture } from "../src/__tests__/helpers/tool-gateway-listing-fixture.js"; +import { stagingShapedToolPolicies } from "../src/__tests__/helpers/tool-policy-index-fixture.js"; +import * as current from "../src/services/tool-access-policy.js"; + +function option(name: string, fallback: string) { + const index = process.argv.indexOf(name); + return index < 0 ? fallback : process.argv[index + 1] ?? fallback; +} +const toolCount = Number(option("--tools", "800")); +const rounds = Number(option("--rounds", "9")); +const baselineRef = option("--baseline-ref", "origin/master"); +const concurrency = 16; // LISTING_DECISION_CONCURRENCY in tool-gateway.ts + +// The baseline imports its siblings by relative path, so it sits next to the +// current service for the run. Each run uses its own file, so overlapping runs +// cannot replace or remove another run's baseline. +const baselineSource = execFileSync("git", ["show", `${baselineRef}:server/src/services/tool-access-policy.ts`], { + cwd: fileURLToPath(new URL("..", import.meta.url)), maxBuffer: 64 * 1024 * 1024, +}); +const baselineUrl = new URL( + `../src/services/tool-access-policy.benchmark-baseline-${process.pid}-${randomBytes(4).toString("hex")}.ts`, + import.meta.url, +); +type Service = typeof current.toolAccessPolicyService; +let baseline: { toolAccessPolicyService: Service; createToolAccessDecisionCache: typeof current.createToolAccessDecisionCache }; +try { + writeFileSync(baselineUrl, baselineSource, { flag: "wx" }); + baseline = await import(pathToFileURL(fileURLToPath(baselineUrl)).href); +} finally { + rmSync(baselineUrl, { force: true }); +} + +async function mapWithConcurrency(items: readonly T[], mapper: (item: T) => Promise) { + const results = new Array(items.length); + let next = 0; + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, async () => { + while (next < items.length) { + const index = next++; + results[index] = await mapper(items[index]!); + } + })); + return results; +} + +type Db = ReturnType; + +/** Counts reads of the raw selectors on every tool_policies row. */ +function countingDb(db: Db) { + const counter = { reads: 0, selectorMatches: 0 }; + const counted = new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields); + const from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args); + if (args[0] !== toolPolicies) return selection; + const execute = selection.execute.bind(selection); + selection.execute = (async (...rest: Parameters) => (await execute(...rest) as Array) + .map((row) => ({ ...row, selectors: new Proxy(row.selectors, { + get(selector, property, receiver) { counter.reads += 1; return Reflect.get(selector, property, receiver); }, + // A raw selector match starts with Object.keys(selector). + ownKeys(selector) { counter.reads += 1; counter.selectorMatches += 1; return Reflect.ownKeys(selector); }, + }) }))) as typeof execute; + return selection; + }) as typeof builder.from; + return builder; + }; + } }); + return { db: counted, counter }; +} + +const database = await startEmbeddedPostgresTestDatabase("paperclip-policy-matching-benchmark-"); +const db = createDb(database.connectionString); +try { + const fixture = await createListingFixture(db, toolCount); + await db.insert(toolPolicies).values(stagingShapedToolPolicies({ + catalogEntries: fixture.entries, + agentId: fixture.agent.id, + projectId: fixture.project.id, + applicationId: fixture.application.id, + connectionId: fixture.connection.id, + }).map((values) => ({ ...values, companyId: fixture.company.id }))); + const inputs: ToolAccessDecisionInput[] = fixture.entries.map((entry) => ({ + companyId: fixture.company.id, + actor: { actorType: "agent", actorId: fixture.agent.id, agentId: fixture.agent.id }, + runContext: { heartbeatRunId: fixture.run.id, issueId: fixture.issue.id, projectId: fixture.project.id, gatewayId: fixture.namedGateway.id }, + request: { + toolName: entry.toolName!, applicationId: fixture.application.id, connectionId: fixture.connection.id, + catalogEntryId: entry.id, upstreamToolName: entry.toolName!, riskLevel: entry.riskLevel, arguments: {}, + }, + consumeRateLimit: false, + })); + const policyRows = await db.select().from(toolPolicies); + const implementations = { + baseline: { service: baseline.toolAccessPolicyService(db), cache: baseline.createToolAccessDecisionCache }, + current: { service: current.toolAccessPolicyService(db), cache: current.createToolAccessDecisionCache }, + }; + const listing = async (name: keyof typeof implementations) => { + const { service, cache } = implementations[name]; + const decisionCache = cache(); + return mapWithConcurrency(inputs, (input) => service.decide(input, { cache: decisionCache })); + }; + + // Same decisions from both before measuring. + const [before, after] = [await listing("baseline"), await listing("current")]; + if (JSON.stringify(before) !== JSON.stringify(after)) throw new Error("Baseline and current decisions differ"); + + // Selector work per listing, measured on the raw rows. + const work: Record = {}; + for (const [name, factory] of [["baseline", baseline.toolAccessPolicyService], ["current", current.toolAccessPolicyService]] as const) { + const { db: counted, counter } = countingDb(db); + const service = factory(counted); + const decisionCache = current.createToolAccessDecisionCache(); + await mapWithConcurrency(inputs, (input) => service.decide(input, { cache: decisionCache })); + work[name] = { rawSelectorReads: counter.reads, selectorEvaluations: counter.selectorMatches }; + } + // The current path evaluates compiled selectors, one per candidate policy. + const index = current.indexEnabledToolPolicies(policyRows); + work.current!.selectorEvaluations = fixture.entries.reduce((sum, entry) => sum + index.candidates({ + actorType: "agent", agentId: null, projectId: null, routineId: null, issueId: null, gatewayId: null, applicationId: null, + connectionId: null, catalogEntryId: entry.id, applicationKey: null, providerType: null, toolName: entry.toolName!, + upstreamToolName: entry.toolName!, riskLevel: null, + }).length, 0); + + const samples: Record> = { baseline: [], current: [] }; + for (let round = 0; round < rounds + 2; round += 1) { + const order = round % 2 === 0 ? ["baseline", "current"] as const : ["current", "baseline"] as const; + for (const name of order) { + global.gc?.(); + const delay = monitorEventLoopDelay({ resolution: 1 }); + // The delay timer runs before the timed work starts and after it ends, so + // it also records a long final synchronous stretch. + delay.enable(); + await sleep(5); + const cpu = process.cpuUsage(); + const started = performance.now(); + await listing(name); + const wallMs = performance.now() - started; + const used = process.cpuUsage(cpu); + await sleep(5); + delay.disable(); + // Two warm-up rounds are not recorded. + if (round >= 2) samples[name]!.push({ wallMs, cpuMs: (used.user + used.system) / 1000, maxBlockMs: delay.max / 1e6 }); + } + } + const stats = (values: number[]) => { + const sorted = [...values].sort((left, right) => left - right); + const round1 = (value: number) => Math.round(value * 10) / 10; + return { median: round1(sorted[Math.floor(sorted.length / 2)]!), min: round1(sorted[0]!), max: round1(sorted.at(-1)!) }; + }; + for (const name of ["baseline", "current"] as const) { + const values = samples[name]!; + console.log(JSON.stringify({ + implementation: name, tools: toolCount, policies: policyRows.length, runs: values.length, + wallMs: stats(values.map((sample) => sample.wallMs)), + cpuMs: stats(values.map((sample) => sample.cpuMs)), + maxEventLoopBlockMs: stats(values.map((sample) => sample.maxBlockMs)), + ...work[name], + })); + } +} finally { + await database.cleanup(); +} diff --git a/server/src/__tests__/helpers/tool-policy-index-fixture.ts b/server/src/__tests__/helpers/tool-policy-index-fixture.ts new file mode 100644 index 00000000000..28a0bc1a889 --- /dev/null +++ b/server/src/__tests__/helpers/tool-policy-index-fixture.ts @@ -0,0 +1,87 @@ +import type { toolPolicies } from "@paperclipai/db"; + +type PolicyValues = Pick; + +/** + * Enabled policies shaped like the staging company whose tools/list profile + * showed policy matching on the main thread: 363 policies, of which 342 select + * only a catalog entry, 18 a catalog entry and its connection, 3 several other + * keys, and 17 carry conditions. 359 require approval, 3 are trust rules and 1 + * blocks. + */ +export function stagingShapedToolPolicies(input: { + catalogEntries: ReadonlyArray<{ id: string; connectionId: string | null; toolName: string | null }>; + agentId: string; + projectId: string; + applicationId: string; + connectionId: string; +}): PolicyValues[] { + const { catalogEntries } = input; + if (catalogEntries.length === 0) throw new Error("At least one catalog entry is required"); + const entry = (index: number) => catalogEntries[index % catalogEntries.length]!; + const conditionShapes: Array> = [ + { risk: { isWrite: true } }, + { arguments: { fieldExists: ["query"] } }, + { context: { requireIssue: true } }, + { timeWindow: { daysOfWeekUtc: [0, 1, 2, 3, 4, 5, 6] } }, + ]; + const policies: PolicyValues[] = []; + for (let index = 0; index < 342; index += 1) { + policies.push({ + name: `Ask first ${index}`, + policyType: "require_approval", + priority: 100, + selectors: { catalogEntryId: entry(index).id }, + conditions: index < 17 ? conditionShapes[index % conditionShapes.length]! : null, + config: { source: "connection_intent" }, + }); + } + for (let index = 0; index < 18; index += 1) { + const selected = entry(342 + index); + policies.push({ + name: `Ask first on connection ${index}`, + policyType: "require_approval", + priority: 100, + selectors: { catalogEntryId: selected.id, connectionId: selected.connectionId ?? input.connectionId }, + conditions: null, + config: null, + }); + } + const trustedTool = entry(0).toolName ?? "tool"; + policies.push( + { + name: "Trusted for agent", + policyType: "trust_rule", + priority: 50, + selectors: { agentId: input.agentId, applicationId: input.applicationId, connectionId: input.connectionId, toolName: trustedTool }, + conditions: null, + config: { trustRule: { argumentFilters: { fieldEquals: { query: "status" } }, approvalThreshold: 3 } }, + }, + { + name: "Trusted in project", + policyType: "trust_rule", + priority: 50, + selectors: { projectId: input.projectId, connectionId: input.connectionId, toolName: trustedTool }, + conditions: null, + config: { trustRule: { argumentFilters: { fieldEquals: { query: "list" } }, approvalThreshold: 3 } }, + }, + { + name: "Trusted on application", + policyType: "trust_rule", + priority: 50, + selectors: { applicationId: input.applicationId, agentId: input.agentId, toolName: entry(1).toolName ?? "tool" }, + conditions: null, + config: { trustRule: { argumentFilters: { fieldEquals: { query: "read" } }, approvalThreshold: 3 } }, + }, + ); + // The block replaces one ask-first selector so the counts stay at 363. + policies[341] = { + name: "Block one action", + policyType: "block", + priority: 10, + selectors: { catalogEntryId: entry(341).id }, + conditions: null, + config: null, + }; + return policies; +} diff --git a/server/src/__tests__/tool-access-policy-index.test.ts b/server/src/__tests__/tool-access-policy-index.test.ts new file mode 100644 index 00000000000..1eb798ddd78 --- /dev/null +++ b/server/src/__tests__/tool-access-policy-index.test.ts @@ -0,0 +1,419 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createDb, toolPolicies, type Db as DbClient } from "@paperclipai/db"; +import { toolPolicyConditionsSchema, type ToolAccessDecisionInput } from "@paperclipai/shared"; +import { + createToolAccessDecisionCache, + evaluatePolicyConditions, + indexEnabledToolPolicies, + matchEnabledToolPolicies, + policyConditions, + selectorMatches, + toolAccessPolicyService, + type EnabledToolPolicyMatch, + type ToolAccessContext, +} from "../services/tool-access-policy.js"; +import { compilePolicySelector } from "../services/tool-access/policy-selector-index.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createListingFixture } from "./helpers/tool-gateway-listing-fixture.js"; +import { stagingShapedToolPolicies } from "./helpers/tool-policy-index-fixture.js"; + +type PolicyRow = typeof toolPolicies.$inferSelect; + +/** The unindexed matching that decisions used before the policy index. */ +function unindexedMatch(policies: PolicyRow[], ctx: ToolAccessContext): EnabledToolPolicyMatch { + for (const policy of policies) { + const conditions = policyConditions(policy); + if (conditions && selectorMatches(policy.selectors, ctx)) { + const parsed = toolPolicyConditionsSchema.safeParse(conditions); + if (!parsed.success) { + return { + unsupportedConditions: { + policy, + issues: parsed.error.issues.map((issue) => ({ path: issue.path.join("."), message: issue.message })), + }, + }; + } + } + } + return { + matching: policies + .map((policy) => ({ policy, conditionEvaluation: evaluatePolicyConditions(policyConditions(policy), ctx) })) + .filter(({ policy, conditionEvaluation }) => selectorMatches(policy.selectors, ctx) && conditionEvaluation.matched), + }; +} + +function seeded(seed: number) { + let state = seed >>> 0; + const next = () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + const pick = (values: readonly T[]): T => values[Math.floor(next() * values.length)]!; + return { next, pick }; +} + +const ids = (count: number) => Array.from({ length: count }, () => randomUUID()); +const catalogIds = ids(30); +const connectionIds = ids(3); +const applicationIds = ids(2); +const agentIds = ids(3); +const projectIds = ids(2); +const toolNames = ["search", "deploy", "read_file", "write_file", "delete_repo", "list_issues"]; +const upstreamNames = ["upstream_search", "deploy", "upstream_delete", null]; +const riskLevels = ["read", "write", "destructive", "low", "high", null] as const; +const policyTypes = ["require_approval", "block", "trust_rule", "rate_limit", "allow", "redact"] as const; + +function policyRow(index: number, values: Partial): PolicyRow { + return { + id: randomUUID(), + companyId: "company", + name: `Policy ${index}`, + description: null, + policyType: "require_approval", + priority: 100, + enabled: true, + selectors: {}, + conditions: null, + config: null, + createdByAgentId: null, + createdByUserId: null, + createdAt: new Date(Date.UTC(2026, 0, 1, 0, 0, index)), + updatedAt: new Date(Date.UTC(2026, 0, 1, 0, 0, index)), + ...values, + }; +} + +function randomSelector(random: ReturnType): Record { + const { next, pick } = random; + const roll = next(); + if (roll < 0.35) return { catalogEntryId: pick(catalogIds) }; + if (roll < 0.45) return { catalogEntryId: pick(catalogIds), connectionId: pick(connectionIds) }; + if (roll < 0.5) { const id = pick(catalogIds); return { catalogEntryIds: [id, pick(catalogIds), id] }; } + if (roll < 0.53) return { catalogEntryIds: ` ${pick(catalogIds)} ` }; + if (roll < 0.56) return { catalogEntryId: "", toolName: pick(toolNames) }; + if (roll < 0.6) return { toolNames: [pick(toolNames), pick(upstreamNames.filter(Boolean))] }; + if (roll < 0.64) return { toolName: pick([...toolNames, "upstream_search"]), riskLevel: pick(riskLevels.filter(Boolean)) }; + if (roll < 0.67) return {}; + if (roll < 0.7) return { riskLevels: ["write", "destructive"] }; + if (roll < 0.73) return { actorType: "agent", agentIds: [pick(agentIds), pick(agentIds)] }; + if (roll < 0.75) return { catalogEntryId: 42, toolNames: [] }; + if (roll < 0.77) return { catalogEntryId: pick(catalogIds), catalogEntryIds: [pick(catalogIds)] }; + if (roll < 0.79) return { catalogEntryIds: ["", " ", pick(catalogIds)] }; + if (roll < 0.81) return { toolName: "", projectId: pick(projectIds) }; + // Several other keys, with no catalog entry or tool name. + const selector: Record = {}; + if (next() < 0.5) selector.agentId = pick(agentIds); + if (next() < 0.5) selector.applicationIds = [pick(applicationIds)]; + if (next() < 0.5) selector.connectionId = pick(connectionIds); + if (next() < 0.5) selector.projectIds = projectIds; + if (next() < 0.3) selector.providerType = pick(["mcp_remote_http", "paperclip_self"]); + if (next() < 0.3) selector.applicationKeys = "github"; + if (next() < 0.3) selector.gatewayId = "gateway-1"; + if (next() < 0.3) selector.issueIds = ["issue-1", "issue-2"]; + if (next() < 0.3) selector.routineId = "routine-1"; + return selector; +} + +function randomConditions(random: ReturnType): Record | null { + const { next, pick } = random; + if (next() < 0.75) return null; + return pick>([ + { risk: { isWrite: true } }, + { risk: { max: "read" } }, + { arguments: { fieldExists: ["query"] } }, + { arguments: { fieldEquals: { mode: "dry_run" } } }, + { context: { requireIssue: true } }, + { actor: { actorTypes: ["agent"] } }, + { trustBoundary: { remoteHttpOnly: true } }, + { timeWindow: { daysOfWeekUtc: [0, 1, 2, 3, 4, 5, 6] } }, + { timeWindow: { startAt: "2999-01-01T00:00:00Z" } }, + // Conditions the runtime cannot evaluate deny a selected call. + { unknownGroup: { enabled: true } }, + { arguments: { fieldEquals: {} } }, + ]); +} + +function randomPolicies(seed: number, count: number): PolicyRow[] { + const random = seeded(seed); + return Array.from({ length: count }, (_, index) => policyRow(index, { + // "redact" is a legacy type the runtime no longer supports. + policyType: random.pick(policyTypes) as PolicyRow["policyType"], + priority: random.pick([10, 50, 100, 100, 100, 200]), + selectors: randomSelector(random), + conditions: randomConditions(random), + })).sort((left, right) => left.priority - right.priority || left.createdAt.getTime() - right.createdAt.getTime()); +} + +function randomContext(random: ReturnType): ToolAccessContext { + const { next, pick } = random; + return { + companyId: "company", + actorType: pick(["agent", "agent", "user", "system"] as const), + actorId: "actor", + agentId: next() < 0.8 ? pick(agentIds) : null, + heartbeatRunId: null, + issueId: pick(["issue-1", "issue-3", null]), + projectId: next() < 0.7 ? pick(projectIds) : null, + routineId: pick(["routine-1", null]), + gatewayId: pick(["gateway-1", null]), + applicationId: pick([...applicationIds, null]), + connectionId: next() < 0.9 ? pick(connectionIds) : null, + // Some contexts name catalog entries that no policy selects, or none. + catalogEntryId: next() < 0.85 ? pick(catalogIds) : pick([randomUUID(), null, ""]), + catalogStatus: "active", + catalogVersionHash: null, + catalogSchemaHash: null, + providerType: pick(["mcp_remote_http", "paperclip_self", null]), + applicationKey: pick(["github", "linear", null]), + upstreamToolName: pick(upstreamNames), + toolName: pick([...toolNames, ""]), + riskLevel: pick(riskLevels), + argumentsHash: "hash", + arguments: pick([{}, { query: "x" }, { mode: "dry_run" }]), + }; +} + +function summary(match: EnabledToolPolicyMatch) { + if ("unsupportedConditions" in match) { + return { unsupported: match.unsupportedConditions.policy.id, issues: match.unsupportedConditions.issues }; + } + return { matching: match.matching.map(({ policy, conditionEvaluation }) => ({ id: policy.id, conditionEvaluation })) }; +} + +describe("tool policy index", () => { + it("compiles selectors that match exactly the contexts the raw selector matches", () => { + const random = seeded(7); + const selectors = [ + ...Array.from({ length: 400 }, () => randomSelector(random)), + { catalogEntryIds: " " }, + { toolName: 7, toolNames: "deploy" }, + { actorTypes: "agent" }, + { riskLevel: "" }, + ]; + let compared = 0; + for (let index = 0; index < 300; index += 1) { + const ctx = randomContext(random); + for (const selector of selectors) { + expect(compilePolicySelector(selector).matches(ctx)).toBe(selectorMatches(selector, ctx)); + compared += 1; + } + } + expect(compared).toBe(300 * selectors.length); + }); + + it("selects the same policies, in the same order, as unindexed matching", () => { + let unsupported = 0; + let matched = 0; + for (const seed of [1, 2, 3, 4, 5]) { + const policies = randomPolicies(seed, 400); + const index = indexEnabledToolPolicies(policies); + const random = seeded(seed * 101); + for (let call = 0; call < 400; call += 1) { + const ctx = randomContext(random); + const expected = summary(unindexedMatch(policies, ctx)); + // A reused index must give the same answer as a fresh one. + expect(summary(matchEnabledToolPolicies(index, ctx))).toEqual(expected); + expect(summary(matchEnabledToolPolicies(indexEnabledToolPolicies(policies), ctx))).toEqual(expected); + if ("unsupported" in expected) unsupported += 1; + else if (expected.matching.length > 0) matched += 1; + } + } + // The generated cases exercise both outcomes. + expect(unsupported).toBeGreaterThan(50); + expect(matched).toBeGreaterThan(250); + }); + + it("selects the same policies as unindexed matching for a staging-shaped set", () => { + const random = seeded(11); + const catalogEntries = Array.from({ length: 800 }, (_, index) => ({ + id: randomUUID(), connectionId: connectionIds[index % connectionIds.length]!, toolName: `tool_${index}`, + })); + const policies = stagingShapedToolPolicies({ + catalogEntries, agentId: agentIds[0]!, projectId: projectIds[0]!, applicationId: applicationIds[0]!, connectionId: connectionIds[0]!, + }).map((values, position) => policyRow(position, { + ...values, priority: values.priority ?? 100, selectors: values.selectors ?? {}, conditions: values.conditions ?? null, config: values.config ?? null, + })).sort((left, right) => left.priority - right.priority || left.createdAt.getTime() - right.createdAt.getTime()); + const index = indexEnabledToolPolicies(policies); + let matched = 0; + for (const [position, entry] of catalogEntries.entries()) { + const ctx: ToolAccessContext = { + ...randomContext(random), + agentId: agentIds[0]!, projectId: projectIds[0]!, applicationId: applicationIds[0]!, + connectionId: entry.connectionId, catalogEntryId: entry.id, toolName: entry.toolName, + upstreamToolName: position % 2 === 0 ? entry.toolName : null, + }; + const expected = summary(unindexedMatch(policies, ctx)); + expect(summary(matchEnabledToolPolicies(index, ctx))).toEqual(expected); + if ((expected.matching?.length ?? 0) > 0) matched += 1; + } + expect(matched).toBeGreaterThan(300); + }); + + it("applies an earlier policy from another index bucket before a catalog-entry policy", () => { + const catalogEntryId = randomUUID(); + const otherCatalogEntryId = randomUUID(); + const policies = [ + // No catalog entry: indexed by tool name, matched through the upstream name. + policyRow(0, { priority: 5, policyType: "require_approval", selectors: { toolNames: ["upstream_deploy"] } }), + // Neither key: a candidate for every call. + policyRow(1, { priority: 6, policyType: "trust_rule", selectors: { riskLevel: "write" } }), + // An empty catalog entry selector does not constrain the call. + policyRow(2, { priority: 7, policyType: "allow", selectors: { catalogEntryId: "", toolName: "deploy" } }), + policyRow(3, { priority: 10, policyType: "block", selectors: { catalogEntryId } }), + policyRow(4, { priority: 20, policyType: "require_approval", selectors: { catalogEntryIds: [otherCatalogEntryId, catalogEntryId] } }), + policyRow(5, { priority: 30, policyType: "block", selectors: { catalogEntryId: otherCatalogEntryId } }), + ]; + const ctx: ToolAccessContext = { + ...randomContext(seeded(3)), + catalogEntryId, toolName: "deploy", upstreamToolName: "upstream_deploy", riskLevel: "write", + }; + const match = matchEnabledToolPolicies(indexEnabledToolPolicies(policies), ctx); + expect(summary(match)).toEqual(summary(unindexedMatch(policies, ctx))); + expect("matching" in match && match.matching.map(({ policy }) => policy.name)) + .toEqual(["Policy 0", "Policy 1", "Policy 2", "Policy 3", "Policy 4"]); + + // A selected policy with unsupported conditions denies the call even when + // a catalog-entry policy sits earlier in a different bucket. + const unsupported = [ + policyRow(0, { priority: 1, selectors: { catalogEntryId } }), + policyRow(1, { priority: 2, selectors: { toolName: "deploy" }, conditions: { unknownGroup: true } }), + ]; + expect(matchEnabledToolPolicies(indexEnabledToolPolicies(unsupported), ctx)).toMatchObject({ + unsupportedConditions: { policy: { name: "Policy 1" } }, + }); + }); +}); + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +type Db = ReturnType; + +/** A db handle whose tool_policies rows count every read of their selectors. */ +function countingPolicySelectorReads(db: Db) { + const counter = { reads: 0 }; + const counted = new Proxy(db, { + get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields); + const from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args); + if (args[0] !== toolPolicies) return selection; + const execute = selection.execute.bind(selection); + selection.execute = (async (...executeArgs: Parameters) => { + const rows = await execute(...executeArgs) as PolicyRow[]; + return rows.map((row) => ({ + ...row, + selectors: new Proxy(row.selectors, { + get(selector, property, receiver) { + counter.reads += 1; + return Reflect.get(selector, property, receiver); + }, + ownKeys(selector) { + counter.reads += 1; + return Reflect.ownKeys(selector); + }, + }), + })); + }) as typeof execute; + return selection; + }) as typeof builder.from; + return builder; + }; + }, + }); + return { db: counted, counter }; +} + +describeEmbeddedPostgres("tool policy index decisions", () => { + let db!: Db; + let tempDb: Awaited> | null = null; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-tool-policy-index-"); + db = createDb(tempDb.connectionString); + }, 20_000); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + function decisionInput( + fixture: Awaited>, + entry: Awaited>["entries"][number], + ): ToolAccessDecisionInput { + return { + companyId: fixture.company.id, + actor: { actorType: "agent", actorId: fixture.agent.id, agentId: fixture.agent.id }, + runContext: { heartbeatRunId: fixture.run.id, issueId: fixture.issue.id, projectId: fixture.project.id, gatewayId: fixture.namedGateway.id }, + request: { + toolName: entry.toolName!, applicationId: fixture.application.id, connectionId: fixture.connection.id, + catalogEntryId: entry.id, upstreamToolName: entry.toolName!, riskLevel: entry.riskLevel, arguments: { query: "status" }, + }, + consumeRateLimit: false, + }; + } + + it("reads policy selectors once per decision batch, however many tools it decides", async () => { + const fixture = await createListingFixture(db, 40); + await db.insert(toolPolicies).values(stagingShapedToolPolicies({ + catalogEntries: Array.from({ length: 800 }, (_, index) => fixture.entries[index] ?? { id: randomUUID(), connectionId: null, toolName: null }), + agentId: fixture.agent.id, + projectId: fixture.project.id, + applicationId: fixture.application.id, + connectionId: fixture.connection.id, + }).map((values) => ({ ...values, companyId: fixture.company.id }))); + const { db: counted, counter } = countingPolicySelectorReads(db); + const policy = toolAccessPolicyService(counted); + const cache = createToolAccessDecisionCache(); + + const first = await policy.decide(decisionInput(fixture, fixture.entries[0]!), { cache }); + const readsForOneTool = counter.reads; + const rest = []; + for (const entry of fixture.entries.slice(1)) rest.push(await policy.decide(decisionInput(fixture, entry), { cache })); + + expect(readsForOneTool).toBeGreaterThan(0); + expect(counter.reads).toBe(readsForOneTool); + const uncached = toolAccessPolicyService(db); + const expected = []; + for (const entry of fixture.entries) expected.push(await uncached.decide(decisionInput(fixture, entry))); + expect([first, ...rest]).toEqual(expected); + expect(new Set(expected.map((decision) => decision.reasonCode))).toEqual(new Set([ + "allow_profile", "deny_default", "deny_policy_block", "requires_approval_policy", + ])); + }, 30_000); + + it("lets an earlier tool-name policy decide before a catalog-entry block", async () => { + const fixture = await createListingFixture(db, 6); + const [blocked, written] = [fixture.entries[2]!, fixture.entries[3]!]; + // The fixture blocks entries[2] at priority 10 and asks first for writes at 20. + const [byToolName] = await db.insert(toolPolicies).values({ + companyId: fixture.company.id, + name: `Ask first by tool name ${randomUUID()}`, + description: "Ask before this action.", + policyType: "require_approval", + priority: 5, + selectors: { toolNames: [blocked.toolName!] }, + }).returning(); + const policy = toolAccessPolicyService(db); + for (const cache of [undefined, createToolAccessDecisionCache()]) { + const blockedDecision = await policy.decide(decisionInput(fixture, blocked), { cache }); + expect(blockedDecision).toMatchObject({ + decision: "require_approval", + reasonCode: "requires_approval_policy", + explanation: "Ask before this action.", + matchedPolicyIds: [byToolName!.id], + policyExplanation: { policyId: byToolName!.id, selectorMatched: true, conditionsMatched: [] }, + }); + const writeDecision = await policy.decide(decisionInput(fixture, written), { cache }); + expect(writeDecision).toMatchObject({ decision: "require_approval", reasonCode: "requires_approval_policy" }); + expect(writeDecision.matchedPolicyIds).not.toContain(byToolName!.id); + } + }, 30_000); +}); diff --git a/server/src/services/tool-access-policy.ts b/server/src/services/tool-access-policy.ts index 839c2a43c54..3ee19aa3fd4 100644 --- a/server/src/services/tool-access-policy.ts +++ b/server/src/services/tool-access-policy.ts @@ -54,6 +54,13 @@ import { } from "./tool-profile-binding-precedence.js"; import { recordToolRuntimeAuditWriteFailure } from "./tool-runtime-metrics.js"; import { canonicalToolArguments, readSignedToolArgumentsPayload, toolRequestIdentityHash } from "./tool-content-guards.js"; +import { + compilePolicySelector, + indexPoliciesBySelector, + listValues, + type CompiledPolicySelector, + type PolicySelectorIndex, +} from "./tool-access/policy-selector-index.js"; class RejectedInvocationAdmission extends Error { constructor(readonly decision: ToolAccessDecision) { @@ -61,7 +68,7 @@ class RejectedInvocationAdmission extends Error { } } -type ToolAccessContext = { +export type ToolAccessContext = { companyId: string; actorType: "agent" | "user" | "system" | "plugin"; actorId: string; @@ -399,12 +406,6 @@ function decision( }; } -function listValues(value: unknown): string[] { - if (typeof value === "string" && value.trim()) return [value.trim()]; - if (!Array.isArray(value)) return []; - return value.filter((item): item is string => typeof item === "string" && item.trim().length > 0); -} - function asToolRiskLevel(value: unknown): ToolRiskLevel | null { if ( value === "low" @@ -420,7 +421,7 @@ function asToolRiskLevel(value: unknown): ToolRiskLevel | null { return null; } -function selectorMatches(selector: ToolAccessSelector | Record | null | undefined, ctx: ToolAccessContext): boolean { +export function selectorMatches(selector: ToolAccessSelector | Record | null | undefined, ctx: ToolAccessContext): boolean { if (!selector || Object.keys(selector).length === 0) return true; const s = selector as Record; // Tool/action names are exact selectors. Glob-looking values are treated as literal names. @@ -507,7 +508,7 @@ function timeWindowMatches(condition: Record, now: Date): boole return true; } -function policyConditions(policy: typeof toolPolicies.$inferSelect): ToolPolicyConditions | null { +export function policyConditions(policy: typeof toolPolicies.$inferSelect): ToolPolicyConditions | null { return isRecord(policy.conditions) && Object.keys(policy.conditions).length > 0 ? policy.conditions as ToolPolicyConditions : null; @@ -517,7 +518,7 @@ function conditionGroupFail(group: string, reason: string): PolicyConditionEvalu return { matched: false, matchedGroups: [], failedGroup: group, reason }; } -function evaluatePolicyConditions( +export function evaluatePolicyConditions( conditions: ToolPolicyConditions | null | undefined, ctx: ToolAccessContext, now = new Date(), @@ -623,6 +624,61 @@ function evaluatePolicyConditions( return { matched: true, matchedGroups }; } +type EnabledToolPolicy = { + policy: typeof toolPolicies.$inferSelect; + selector: CompiledPolicySelector; + conditions: ToolPolicyConditions | null; + conditionsCheck?: ReturnType; +}; + +/** + * Compiles a company's enabled policies, already in priority and createdAt + * order, once per decision batch: selectors are parsed once and policies are + * indexed so that each decision checks only the policies that can match it. + */ +export function indexEnabledToolPolicies( + policies: ReadonlyArray, +): PolicySelectorIndex { + return indexPoliciesBySelector( + policies.map((policy) => ({ policy, selector: compilePolicySelector(policy.selectors), conditions: policyConditions(policy) })), + (entry) => entry.selector, + ); +} + +export type EnabledToolPolicyMatch = + | { unsupportedConditions: { policy: typeof toolPolicies.$inferSelect; issues: Array<{ path: string; message: string }> } } + | { matching: Array<{ policy: typeof toolPolicies.$inferSelect; conditionEvaluation: PolicyConditionEvaluation }> }; + +/** + * The enabled policies that apply to one call, in evaluation order. The first + * selected policy with conditions the runtime cannot evaluate wins over every + * match, so the caller denies the call. + */ +export function matchEnabledToolPolicies( + index: PolicySelectorIndex, + ctx: ToolAccessContext, +): EnabledToolPolicyMatch { + const selected = index.candidates(ctx).filter((entry) => entry.selector.matches(ctx)); + for (const entry of selected) { + if (!entry.conditions) continue; + entry.conditionsCheck ??= toolPolicyConditionsSchema.safeParse(entry.conditions); + if (!entry.conditionsCheck.success) { + return { + unsupportedConditions: { + policy: entry.policy, + issues: entry.conditionsCheck.error.issues.map((issue) => ({ path: issue.path.join("."), message: issue.message })), + }, + }; + } + } + const matching = []; + for (const { policy, conditions } of selected) { + const conditionEvaluation = evaluatePolicyConditions(conditions, ctx); + if (conditionEvaluation.matched) matching.push({ policy, conditionEvaluation }); + } + return { matching }; +} + function profileEntryMatches(entry: typeof toolProfileEntries.$inferSelect, ctx: ToolAccessContext): boolean { const selected = entry.selectorType === "application" ? entry.applicationId === ctx.applicationId : entry.selectorType === "connection" ? entry.connectionId === ctx.connectionId @@ -1550,38 +1606,29 @@ export function toolAccessPolicyService(db: Pick entry.effect === "exclude") && (profile.defaultAction === "allow" || matchingEntries.some((entry) => entry.effect === "include")); }); - const policies = await cachedRead(cache, `enabled-policies:${ctx.companyId}`, async () => - db.select().from(toolPolicies).where(and(eq(toolPolicies.companyId, ctx.companyId), eq(toolPolicies.enabled, true))).orderBy(asc(toolPolicies.priority), asc(toolPolicies.createdAt))); - for (const policy of policies) { - const conditions = policyConditions(policy); - if (conditions && selectorMatches(policy.selectors, ctx)) { - const parsed = toolPolicyConditionsSchema.safeParse(conditions); - if (!parsed.success) { - return decision( - "deny", - "deny_policy_block", - "Tool access denied because a matching policy uses unsupported runtime conditions.", - effectiveProfileIds, - [policy.id], - { - redactionPlan: redaction.redactionPlan, - policyExplanation: { - policyId: policy.id, - policyType: policy.policyType, - selectorMatched: true, - conditionsError: parsed.error.issues.map((issue) => ({ - path: issue.path.join("."), - message: issue.message, - })), - }, - }, - ); - } - } + const enabledPolicies = await cachedRead(cache, `enabled-policies:${ctx.companyId}`, async () => indexEnabledToolPolicies( + await db.select().from(toolPolicies).where(and(eq(toolPolicies.companyId, ctx.companyId), eq(toolPolicies.enabled, true))).orderBy(asc(toolPolicies.priority), asc(toolPolicies.createdAt)))); + const policyMatch = matchEnabledToolPolicies(enabledPolicies, ctx); + if ("unsupportedConditions" in policyMatch) { + const { policy, issues } = policyMatch.unsupportedConditions; + return decision( + "deny", + "deny_policy_block", + "Tool access denied because a matching policy uses unsupported runtime conditions.", + effectiveProfileIds, + [policy.id], + { + redactionPlan: redaction.redactionPlan, + policyExplanation: { + policyId: policy.id, + policyType: policy.policyType, + selectorMatched: true, + conditionsError: issues, + }, + }, + ); } - const matchingPolicies = policies - .map((policy) => ({ policy, conditionEvaluation: evaluatePolicyConditions(policyConditions(policy), ctx) })) - .filter(({ policy, conditionEvaluation }) => selectorMatches(policy.selectors, ctx) && conditionEvaluation.matched); + const matchingPolicies = policyMatch.matching; const explicitBlock = matchingPolicies.find(({ policy }) => policy.policyType === "block"); for (const { policy, conditionEvaluation } of matchingPolicies) { const policyExplanation = { diff --git a/server/src/services/tool-access/policy-selector-index.ts b/server/src/services/tool-access/policy-selector-index.ts new file mode 100644 index 00000000000..178f5ab3dcb --- /dev/null +++ b/server/src/services/tool-access/policy-selector-index.ts @@ -0,0 +1,160 @@ +/** + * Compiled tool-policy selectors and the per-company policy index. + * + * A tools/list decision runs once per listed tool. Matching every enabled + * policy's raw selector for every tool made each listing cost tools x policies + * selector checks on the main thread. Compiling the selectors once per decision + * batch and indexing policies by the selector key that rejects most + * (catalogEntryId, then toolName) limits each decision to the policies that can + * match the tool. The compiled matcher has the same semantics as the raw + * selector match in tool-access-policy.ts. + */ + +export type PolicySelectorContext = { + actorType: string; + agentId: string | null; + projectId: string | null; + routineId: string | null; + issueId: string | null; + gatewayId: string | null; + applicationId: string | null; + connectionId: string | null; + catalogEntryId: string | null; + applicationKey: string | null; + providerType: string | null; + toolName: string; + upstreamToolName: string | null; + riskLevel: string | null; +}; + +type ExactKey = Exclude; + +// Selector key pairs matched against one context value each. +const EXACT_SELECTOR_KEYS: ReadonlyArray = [ + ["actorType", "actorTypes"], + ["agentId", "agentIds"], + ["projectId", "projectIds"], + ["routineId", "routineIds"], + ["issueId", "issueIds"], + ["gatewayId", "gatewayIds"], + ["applicationId", "applicationIds"], + ["connectionId", "connectionIds"], + ["catalogEntryId", "catalogEntryIds"], + ["applicationKey", "applicationKeys"], + ["providerType", "providerTypes"], + ["riskLevel", "riskLevels"], +]; + +export function listValues(value: unknown): string[] { + if (typeof value === "string" && value.trim()) return [value.trim()]; + if (!Array.isArray(value)) return []; + return value.filter((item): item is string => typeof item === "string" && item.trim().length > 0); +} + +type Constraint = { single: string | null; many: string[] }; + +export type CompiledPolicySelector = { + matches(ctx: PolicySelectorContext): boolean; + /** Values the context catalogEntryId must equal, or null when unconstrained. */ + readonly catalogEntryIds: readonly string[] | null; + /** Values one of the context tool names must equal, or null when unconstrained. */ + readonly toolNames: readonly string[] | null; +}; + +function constraint(selector: Record, singleKey: string, pluralKey: string): Constraint | null { + const raw = selector[singleKey]; + // An empty single value does not constrain, as in the raw selector match. + const single = typeof raw === "string" && raw ? raw : null; + const many = listValues(selector[pluralKey]); + return single || many.length > 0 ? { single, many } : null; +} + +function indexValues(value: Constraint | null): string[] | null { + if (!value) return null; + return value.single ? [value.single] : [...new Set(value.many)]; +} + +export function compilePolicySelector(selector: unknown): CompiledPolicySelector { + const s: Record = selector && typeof selector === "object" ? selector as Record : {}; + const exact: Array<{ key: ExactKey } & Constraint> = []; + let catalogEntry: Constraint | null = null; + for (const [singleKey, pluralKey] of EXACT_SELECTOR_KEYS) { + const value = constraint(s, singleKey, pluralKey); + if (!value) continue; + exact.push({ key: singleKey, ...value }); + if (singleKey === "catalogEntryId") catalogEntry = value; + } + const toolName = constraint(s, "toolName", "toolNames"); + return { + catalogEntryIds: indexValues(catalogEntry), + toolNames: indexValues(toolName), + matches(ctx) { + for (const { key, single, many } of exact) { + const actual = ctx[key]; + if (single && actual !== single) return false; + if (many.length > 0 && !(actual && many.includes(actual))) return false; + } + if (toolName) { + const { single, many } = toolName; + // Constraint values are never empty, so an empty context name never matches. + const names = [ctx.toolName, ctx.upstreamToolName]; + if (single && !names.includes(single)) return false; + if (many.length > 0 && !many.some((value) => names.includes(value))) return false; + } + return true; + }, + }; +} + +export type PolicySelectorIndex = { + readonly entries: readonly T[]; + /** Entries whose selector can match the context, in their original order. */ + candidates(ctx: PolicySelectorContext): T[]; +}; + +/** + * Indexes entries (already in evaluation order) by catalogEntryId, then by + * toolName for entries without a catalogEntryId selector. Entries with neither + * are candidates for every context. A context that a selector rejects on its + * index key can never match it, so leaving it out changes no decision. + */ +export function indexPoliciesBySelector( + entries: readonly T[], + selectorOf: (entry: T) => CompiledPolicySelector, +): PolicySelectorIndex { + const byCatalogEntryId = new Map(); + const byToolName = new Map(); + const unindexed: number[] = []; + const add = (buckets: Map, keys: readonly string[], position: number) => { + for (const key of keys) { + const bucket = buckets.get(key); + if (bucket) bucket.push(position); + else buckets.set(key, [position]); + } + }; + entries.forEach((entry, position) => { + const selector = selectorOf(entry); + if (selector.catalogEntryIds) add(byCatalogEntryId, selector.catalogEntryIds, position); + else if (selector.toolNames) add(byToolName, selector.toolNames, position); + else unindexed.push(position); + }); + return { + entries, + candidates(ctx) { + const lists: number[][] = []; + if (ctx.catalogEntryId) { + const bucket = byCatalogEntryId.get(ctx.catalogEntryId); + if (bucket) lists.push(bucket); + } + for (const name of new Set([ctx.toolName, ctx.upstreamToolName])) { + const bucket = name ? byToolName.get(name) : undefined; + if (bucket) lists.push(bucket); + } + if (unindexed.length > 0) lists.push(unindexed); + if (lists.length === 0) return []; + if (lists.length === 1) return lists[0]!.map((position) => entries[position]!); + const positions = [...new Set(lists.flat())].sort((left, right) => left - right); + return positions.map((position) => entries[position]!); + }, + }; +}