mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-20 18:26:40 +01:00
Measure database and projection costs in runtime tool discovery (#22347)
## Thinking Path > - Paperclip starts agent turns with instructions for the connections the agent can use. > - The gateway reads the connected-tool catalog during prompt preparation. > - A single discovery span currently covers both database waits and descriptor construction. > - Maintainers need to distinguish these costs before changing catalog reads. > - This change records the transaction, both queries, availability checks, and descriptor construction. ## Linked Issues or Issue Description **Subsystem affected** — Runtime tool discovery and run preparation. **Problem or motivation** — The existing catalog span cannot explain time spent before a remote turn starts. **Proposed solution** — Measure the existing read in place. Record the number of entries and connections returned. **Alternatives considered** — A local microbenchmark cannot reproduce a hosted database or a deployed catalog's sizes. A change to the query would confound the measurement. **Roadmap alignment** — This adds diagnostic detail to an existing operation. Related work includes #14864, which bounded catalog memory use. **Additional context** — The PostgreSQL reads share one read-only snapshot. ## What Changed - Add a transaction span with children for the entry read and the matched-connection read. - Attach the two returned row counts to that transaction. - Time availability checks and descriptor construction under the existing catalog discovery span. - Document which preparation timings can also reach the native run event table. ## Verification - Ran the gateway listing-memory, discovery HTTP, and execution timing context suites with local PostgreSQL. All 30 tests passed. - Checked the parentage and row counts with a real OpenTelemetry exporter in memory. - Ran the standing-instruction integration suite and measured the catalog from two staging application containers with read-only database sessions. ## Risks Each recorded catalog operation adds up to five spans. The existing per-phase bound applies. Only the OTel transaction span includes the aggregate row counts. With or without OTel, the existing native preparation timing path can also persist a run event for each new phase that takes at least 50 ms, subject to its normal caps. The executor defers those writes. The database query, tool eligibility, and authorization contracts stay the same. ## Model Used OpenAI Codex directed and qualified the work. GPT-6.2 Sol implemented the instrumentation and regression test. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge
This commit is contained in:
1 parent
a640b4dac8
commit
28043f8b93
5 files changed
+130
-41
No files matched your search
@@ -1,4 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { context, trace } from "@opentelemetry/api";
|
||||
import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
|
||||
import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
activityLog,
|
||||
@@ -24,6 +27,7 @@ import {
|
||||
createToolAccessDecisionCache,
|
||||
toolAccessPolicyService,
|
||||
} from "../services/tool-access-policy.js";
|
||||
import { withRunExecutionTiming } from "../services/execution-preparation-timing.js";
|
||||
import { createToolGatewayService } from "../services/tool-gateway.js";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
@@ -252,6 +256,48 @@ describeEmbeddedPostgres("tool gateway listing memory", () => {
|
||||
expect(connectionReadParams[0]).not.toContain(emptyConnection.id);
|
||||
});
|
||||
|
||||
it("exports the catalog transaction, query and projection boundaries for a gateway listing", async () => {
|
||||
const fixture = await createListingFixture(db, 20, { connectionCount: 2 });
|
||||
const gateway = createToolGatewayService(db);
|
||||
const exporter = new InMemorySpanExporter();
|
||||
const provider = new TracerProvider({ spanProcessors: [new SimpleSpanProcessor({ exporter })] });
|
||||
const manager = new AsyncLocalStorageContextManager().enable();
|
||||
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test");
|
||||
try {
|
||||
expect(context.setGlobalContextManager(manager)).toBe(true);
|
||||
expect(trace.setGlobalTracerProvider(provider)).toBe(true);
|
||||
const tools = await withRunExecutionTiming(fixture.run.id, () => gateway.listToolsForNamedGateway({
|
||||
gatewayId: fixture.namedGateway.id,
|
||||
bearerToken: fixture.token.token,
|
||||
}));
|
||||
expect(tools.filter((tool) => tool.catalogEntryId)).toHaveLength(18);
|
||||
|
||||
const spans = exporter.getFinishedSpans();
|
||||
const prefix = "connectors.discovery.connections";
|
||||
const connectionPhase = spans.find((span) => span.name === prefix)!;
|
||||
const transaction = spans.find((span) => span.name === `${prefix}.transaction`)!;
|
||||
expect(transaction.parentSpanContext?.spanId).toBe(connectionPhase.spanContext().spanId);
|
||||
expect(transaction.attributes).toMatchObject({
|
||||
"paperclip.execution.catalogEntryCount": 20,
|
||||
"paperclip.execution.connectionCount": 2,
|
||||
});
|
||||
for (const name of ["catalog_entries", "matched_connections"]) {
|
||||
const query = spans.find((span) => span.name === `${transaction.name}.${name}`);
|
||||
expect(query?.parentSpanContext?.spanId).toBe(transaction.spanContext().spanId);
|
||||
}
|
||||
for (const name of ["availability", "descriptors"]) {
|
||||
const phase = spans.find((span) => span.name === `${prefix}.${name}`);
|
||||
expect(phase?.parentSpanContext?.spanId).toBe(connectionPhase.spanContext().spanId);
|
||||
}
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
trace.disable();
|
||||
context.disable();
|
||||
manager.disable();
|
||||
await provider.shutdown();
|
||||
}
|
||||
});
|
||||
|
||||
it("never selects the whole run snapshot or result when it decides access", async () => {
|
||||
// Without agent or company bindings, the project binding is the narrowest match.
|
||||
const fixture = await createListingFixture(db, 3, { broadBindings: false });
|
||||
|
||||
@@ -2,7 +2,7 @@ import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import { withDatabaseOperation } from "@paperclipai/db";
|
||||
import { EXECUTION_PREPARATION_SPAN_ATTRS, getActiveStepContext, runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
|
||||
import { createHash } from "node:crypto";
|
||||
import { isSpanContextValid, ROOT_CONTEXT, SpanStatusCode, trace, type Context } from "@opentelemetry/api";
|
||||
import { isSpanContextValid, ROOT_CONTEXT, SpanStatusCode, trace, type Context, type Span } from "@opentelemetry/api";
|
||||
|
||||
type PreparationPhase = "environment_selection" | "execution_configuration" | "session"
|
||||
| "agent_read" | "chat_completion" | "conversation" | "runtime_state" | "issue_context"
|
||||
@@ -102,6 +102,9 @@ export type ExecutionPhase =
|
||||
| "connectors.browser_use.connection" | "connectors.browser_use.catalog" | "connectors.browser_use.grant"
|
||||
| "connectors.discovery.scheduled" | "connectors.discovery.queue" | "connectors.discovery.build" | "connectors.discovery.agent"
|
||||
| "connectors.discovery.connections" | "connectors.discovery.github" | "connectors.discovery.slack" | "connectors.discovery.speko"
|
||||
| "connectors.discovery.connections.transaction" | "connectors.discovery.connections.transaction.catalog_entries"
|
||||
| "connectors.discovery.connections.transaction.matched_connections"
|
||||
| "connectors.discovery.connections.availability" | "connectors.discovery.connections.descriptors"
|
||||
| "connectors.discovery.policy" | "connectors.discovery.on_demand_policy"
|
||||
| "connectors.slack.candidates" | "connectors.slack.authority" | "connectors.slack.profile"
|
||||
| "connectors.slack.settings" | "connectors.slack.identity" | "connectors.slack.credentials" | "connectors.slack.revision"
|
||||
@@ -124,7 +127,7 @@ export type ExecutionPhase =
|
||||
// recordRunId: the run whose preparation log receives nested measureRunPhase
|
||||
// steps. A traced step opens its own scope, so it carries its preparation
|
||||
// ancestor's target; other scopes (execution attempts) never record.
|
||||
type RunScope = { runId: string; phase: string; parent?: Context; ended: boolean; details: Map<string, number>; recordRunId?: string };
|
||||
type RunScope = { runId: string; phase: string; parent?: Context; span?: Span; ended: boolean; details: Map<string, number>; recordRunId?: string };
|
||||
const ownership = new AsyncLocalStorage<RunScope>();
|
||||
const endpointEnabled = () => Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim());
|
||||
|
||||
@@ -148,6 +151,20 @@ export function recordRepositoryObjectCacheDetail(detail: {
|
||||
} catch {}
|
||||
}
|
||||
|
||||
export function recordConnectionCatalogCounts(catalogEntryCount: number, connectionCount: number): void {
|
||||
const owned = ownership.getStore();
|
||||
if (!endpointEnabled() || owned?.phase !== "connectors.discovery.connections.transaction" || !owned.span || owned.ended) return;
|
||||
try {
|
||||
const span = owned.span;
|
||||
if (Number.isSafeInteger(catalogEntryCount) && catalogEntryCount >= 0) {
|
||||
span.setAttribute(EXECUTION_PREPARATION_SPAN_ATTRS.catalogEntryCount, catalogEntryCount);
|
||||
}
|
||||
if (Number.isSafeInteger(connectionCount) && connectionCount >= 0) {
|
||||
span.setAttribute(EXECUTION_PREPARATION_SPAN_ATTRS.connectionCount, connectionCount);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
export function getRunExecutionTraceContext(runId: string): Context | undefined {
|
||||
const owned = ownership.getStore();
|
||||
return endpointEnabled() && owned?.runId === runId && !owned.ended ? owned.parent : undefined;
|
||||
@@ -209,10 +226,10 @@ function measure<T>(runId: string, name: string, work: () => T | Promise<T>, ind
|
||||
|
||||
function measureTraced<T>(runId: string, name: string, work: () => T | Promise<T>, independent = false, counts: { fileCount?: number; bytes?: number } = {}, activeParent?: Context, recordRunId?: string): Promise<T> {
|
||||
let operation: Promise<T> | undefined;
|
||||
const run = (span?: { setStatus(status: { code: number }): void; end(): void }, parent?: Context) => {
|
||||
const run = (span?: Span, parent?: Context) => {
|
||||
if (operation) return operation;
|
||||
const previous = ownership.getStore();
|
||||
const owned: RunScope = { runId, phase: name,
|
||||
const owned: RunScope = { runId, phase: name, span,
|
||||
parent: parent ?? (!independent && previous?.runId === runId && !previous.ended ? previous.parent : undefined),
|
||||
ended: false,
|
||||
details: !independent && previous?.runId === runId ? previous.details : new Map<string, number>(),
|
||||
|
||||
@@ -31,7 +31,7 @@ import type { BudgetServiceHooks } from "./budgets.js";
|
||||
import type { PromptInjectionDetectorMetadata } from "@paperclipai/shared";
|
||||
import { connectionManagementService, type ConnectionManagementContext } from "./connection-management.js";
|
||||
import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js";
|
||||
import { measureToolDiscovery, measureRunPhase } from "./execution-preparation-timing.js";
|
||||
import { measureToolDiscovery, measureRunPhase, recordConnectionCatalogCounts } from "./execution-preparation-timing.js";
|
||||
import { nativeProviderCatalogFor } from "./native-provider-catalogs.js";
|
||||
import { callGoogleWorkspaceRest, googleWorkspaceRestAuditEndpoint } from "./google-workspace-rest-dispatch.js";
|
||||
import { composeConnectionInstructions } from "./connection-instructions.js";
|
||||
@@ -1837,9 +1837,9 @@ export function createToolGatewayService(
|
||||
// every tool, and one connection can expose hundreds of tools. Both reads
|
||||
// share one read-only snapshot, so they agree like a single join, and the
|
||||
// second read gets only the connections that have listed tools.
|
||||
const { connectionRows, catalogRows } = await db.transaction(
|
||||
const readCatalog = () => db.transaction(
|
||||
async (tx) => {
|
||||
const catalogRows = await tx
|
||||
const catalogRows = await measureRunPhase("connectors.discovery.connections.transaction.catalog_entries", async () => await tx
|
||||
.select({ catalogEntry: toolCatalogEntries })
|
||||
.from(toolCatalogEntries)
|
||||
.innerJoin(
|
||||
@@ -1860,7 +1860,7 @@ export function createToolGatewayService(
|
||||
selectedConnectionIds ? inArray(toolConnections.id, selectedConnectionIds) : undefined,
|
||||
),
|
||||
)
|
||||
.orderBy(toolConnections.name, toolCatalogEntries.name);
|
||||
.orderBy(toolConnections.name, toolCatalogEntries.name));
|
||||
const connectionIds = [
|
||||
...new Set(
|
||||
catalogRows.map(({ catalogEntry }) => catalogEntry.connectionId),
|
||||
@@ -1868,7 +1868,7 @@ export function createToolGatewayService(
|
||||
];
|
||||
if (connectionIds.length === 0)
|
||||
return { connectionRows: [], catalogRows };
|
||||
const connectionRows = await tx
|
||||
const connectionRows = await measureRunPhase("connectors.discovery.connections.transaction.matched_connections", async () => await tx
|
||||
.select({
|
||||
connection: toolConnections,
|
||||
application: toolApplications,
|
||||
@@ -1883,44 +1883,59 @@ export function createToolGatewayService(
|
||||
inArray(toolConnections.id, connectionIds),
|
||||
connectedMcpConnectionFilter(companyId),
|
||||
),
|
||||
);
|
||||
));
|
||||
return { connectionRows, catalogRows };
|
||||
},
|
||||
{ isolationLevel: "repeatable read", accessMode: "read only" },
|
||||
);
|
||||
const connectionRowsById = new Map(
|
||||
connectionRows.map((row) => [row.connection.id, row]),
|
||||
);
|
||||
const rows = catalogRows.flatMap(({ catalogEntry }) => {
|
||||
const connectionRow = connectionRowsById.get(catalogEntry.connectionId);
|
||||
return connectionRow ? [{ catalogEntry, ...connectionRow }] : [];
|
||||
const { connectionRows, catalogRows } = await measureRunPhase("connectors.discovery.connections.transaction", async () => {
|
||||
const rows = await readCatalog();
|
||||
recordConnectionCatalogCounts(rows.catalogRows.length, rows.connectionRows.length);
|
||||
return rows;
|
||||
});
|
||||
const eligibleRows = await measureRunPhase("connectors.discovery.connections.availability", async () => {
|
||||
const connectionRowsById = new Map(
|
||||
connectionRows.map((row) => [row.connection.id, row]),
|
||||
);
|
||||
const rows = catalogRows.flatMap(({ catalogEntry }) => {
|
||||
const connectionRow = connectionRowsById.get(catalogEntry.connectionId);
|
||||
return connectionRow ? [{ catalogEntry, ...connectionRow }] : [];
|
||||
});
|
||||
|
||||
// A plugin connector's tools reach agents only while its plugin is ready
|
||||
// and still declares the app. Disabled plugins keep their saved setup.
|
||||
const pluginApps = await pluginAppCatalogFor(db, connectionRows.map(({ connection }) => connection));
|
||||
const featureSettings = connectionRows.some(({ application }) => getAppDefinitionForApplicationKey(application.applicationKey)?.experimentalFeatureKey)
|
||||
? await instanceSettingsService(db).getExperimental() : undefined;
|
||||
const eligibleRows = rows.filter(
|
||||
({ catalogEntry, connection, application }) =>
|
||||
pluginApps.isAvailable(connectionSourceSlug(connection)) &&
|
||||
(!featureSettings || isConnectorFeatureEnabled(getAppDefinitionForApplicationKey(application.applicationKey), featureSettings)) &&
|
||||
!isRetiredComposioConnection(connection) &&
|
||||
!(nativeProviderCatalogFor(connection) && !nativeProviderCatalogFor(connection)!.isCurrentEntry(catalogEntry)) &&
|
||||
!(restConnectorFor(connection) && !restConnectorFor(connection)!.isCurrent(catalogEntry)) &&
|
||||
!(isRailwayEndpoint(connection.config.url) && (isRailwayToolBlocked(catalogEntry.toolName) || (normalizeRailwayToolName(catalogEntry.toolName).startsWith(RAILWAY_TOOL_PREFIX) && connection.config.railwayApiStatus !== "available"))) &&
|
||||
(((isBrowserUseConnection(connection) || nativeProviderCatalogFor(connection) || restConnectorFor(connection)) && application.type === "rest_api") ||
|
||||
(connection.transport === "mcp_remote" &&
|
||||
application.type === "mcp_http") ||
|
||||
(connection.transport === "local_stdio" &&
|
||||
application.type === "mcp_stdio") ||
|
||||
// Legacy top-level plugin tools keep their original names and are
|
||||
// listed by the plugin tool path, gated by this connection's installs.
|
||||
(connection.transport === "plugin" &&
|
||||
application.type === "paperclip_plugin" &&
|
||||
!pluginConnectionIdentity(connection)?.legacyTools &&
|
||||
pluginCatalogEntryCurrent(connection, catalogEntry))),
|
||||
);
|
||||
// A plugin connector's tools reach agents only while its plugin is ready
|
||||
// and still declares the app. Disabled plugins keep their saved setup.
|
||||
const pluginApps = await pluginAppCatalogFor(db, connectionRows.map(({ connection }) => connection));
|
||||
const featureSettings = connectionRows.some(({ application }) => getAppDefinitionForApplicationKey(application.applicationKey)?.experimentalFeatureKey)
|
||||
? await instanceSettingsService(db).getExperimental() : undefined;
|
||||
return rows.filter(
|
||||
({ catalogEntry, connection, application }) =>
|
||||
pluginApps.isAvailable(connectionSourceSlug(connection)) &&
|
||||
(!featureSettings || isConnectorFeatureEnabled(getAppDefinitionForApplicationKey(application.applicationKey), featureSettings)) &&
|
||||
!isRetiredComposioConnection(connection) &&
|
||||
!(nativeProviderCatalogFor(connection) && !nativeProviderCatalogFor(connection)!.isCurrentEntry(catalogEntry)) &&
|
||||
!(restConnectorFor(connection) && !restConnectorFor(connection)!.isCurrent(catalogEntry)) &&
|
||||
!(isRailwayEndpoint(connection.config.url) && (isRailwayToolBlocked(catalogEntry.toolName) || (normalizeRailwayToolName(catalogEntry.toolName).startsWith(RAILWAY_TOOL_PREFIX) && connection.config.railwayApiStatus !== "available"))) &&
|
||||
(((isBrowserUseConnection(connection) || nativeProviderCatalogFor(connection) || restConnectorFor(connection)) && application.type === "rest_api") ||
|
||||
(connection.transport === "mcp_remote" &&
|
||||
application.type === "mcp_http") ||
|
||||
(connection.transport === "local_stdio" &&
|
||||
application.type === "mcp_stdio") ||
|
||||
// Legacy top-level plugin tools keep their original names and are
|
||||
// listed by the plugin tool path, gated by this connection's installs.
|
||||
(connection.transport === "plugin" &&
|
||||
application.type === "paperclip_plugin" &&
|
||||
!pluginConnectionIdentity(connection)?.legacyTools &&
|
||||
pluginCatalogEntryCurrent(connection, catalogEntry))),
|
||||
);
|
||||
});
|
||||
return measureRunPhase("connectors.discovery.connections.descriptors", () => buildConnectedCatalogDescriptors(eligibleRows));
|
||||
}
|
||||
|
||||
function buildConnectedCatalogDescriptors(eligibleRows: Array<{
|
||||
catalogEntry: typeof toolCatalogEntries.$inferSelect;
|
||||
connection: typeof toolConnections.$inferSelect;
|
||||
application: typeof toolApplications.$inferSelect;
|
||||
}>): ToolGatewayDescriptor[] {
|
||||
const baseNames = eligibleRows.map(
|
||||
({ catalogEntry, connection, application }) => {
|
||||
const toolSlug = slugSegment(catalogEntry.toolName, "tool");
|
||||
|
||||
Reference in new issue
Block a user