diff --git a/doc/heartbeat-internals.md b/doc/heartbeat-internals.md index cd7f237ea79..5e784cd9d86 100644 --- a/doc/heartbeat-internals.md +++ b/doc/heartbeat-internals.md @@ -199,6 +199,15 @@ The executor receives credential ownership and failure-redaction updates as soon as they are available, so later setup failures still clean up acquired resources. Busy-account deferral returns explicitly before session/workspace preparation. +During standing-instruction resolution, the gateway's [OpenTelemetry trace](observability.md) +separates the catalog transaction and its two reads from availability checks +and tool descriptor construction. The transaction span carries the number of +catalog entries and connections returned. With or without an OTLP endpoint, +native run preparation also buffers phase timings of 50 ms or longer, subject +to the existing per-run limits. At dispatch, the native executor queues these +timings for the [instance run log](run-log-events.md). The catalog counts are +trace attributes only. + On Cloud-managed sandbox connections, an ACPX platform check can defer runtime preparation to the normal Runner launch. After the selected account completes its provider hello probe or live API-key check, that specific deferred check is diff --git a/packages/adapter-utils/src/acpx-engine/startup-timing.ts b/packages/adapter-utils/src/acpx-engine/startup-timing.ts index 5ce26855ef2..c71968dd957 100644 --- a/packages/adapter-utils/src/acpx-engine/startup-timing.ts +++ b/packages/adapter-utils/src/acpx-engine/startup-timing.ts @@ -51,6 +51,8 @@ export const EXECUTION_PREPARATION_SPAN_ATTRS = { runId: "paperclip.task.run.run_id", fileCount: "paperclip.execution.fileCount", bytes: "paperclip.execution.bytes", + catalogEntryCount: "paperclip.execution.catalogEntryCount", + connectionCount: "paperclip.execution.connectionCount", repositoryCacheOutcome: "paperclip.execution.repositoryCache.outcome", repositoryCacheReferenceSupplied: "paperclip.execution.repositoryCache.reference_supplied", } as const; diff --git a/server/src/__tests__/tool-gateway-listing-memory.test.ts b/server/src/__tests__/tool-gateway-listing-memory.test.ts index 24d20cdc9fc..d104d0fe5ed 100644 --- a/server/src/__tests__/tool-gateway-listing-memory.test.ts +++ b/server/src/__tests__/tool-gateway-listing-memory.test.ts @@ -1,4 +1,7 @@ import { randomUUID } from "node:crypto"; +import { context, trace } from "@opentelemetry/api"; +import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; +import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { activityLog, @@ -24,6 +27,7 @@ import { createToolAccessDecisionCache, toolAccessPolicyService, } from "../services/tool-access-policy.js"; +import { withRunExecutionTiming } from "../services/execution-preparation-timing.js"; import { createToolGatewayService } from "../services/tool-gateway.js"; import { getEmbeddedPostgresTestSupport, @@ -252,6 +256,48 @@ describeEmbeddedPostgres("tool gateway listing memory", () => { expect(connectionReadParams[0]).not.toContain(emptyConnection.id); }); + it("exports the catalog transaction, query and projection boundaries for a gateway listing", async () => { + const fixture = await createListingFixture(db, 20, { connectionCount: 2 }); + const gateway = createToolGatewayService(db); + const exporter = new InMemorySpanExporter(); + const provider = new TracerProvider({ spanProcessors: [new SimpleSpanProcessor({ exporter })] }); + const manager = new AsyncLocalStorageContextManager().enable(); + vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test"); + try { + expect(context.setGlobalContextManager(manager)).toBe(true); + expect(trace.setGlobalTracerProvider(provider)).toBe(true); + const tools = await withRunExecutionTiming(fixture.run.id, () => gateway.listToolsForNamedGateway({ + gatewayId: fixture.namedGateway.id, + bearerToken: fixture.token.token, + })); + expect(tools.filter((tool) => tool.catalogEntryId)).toHaveLength(18); + + const spans = exporter.getFinishedSpans(); + const prefix = "connectors.discovery.connections"; + const connectionPhase = spans.find((span) => span.name === prefix)!; + const transaction = spans.find((span) => span.name === `${prefix}.transaction`)!; + expect(transaction.parentSpanContext?.spanId).toBe(connectionPhase.spanContext().spanId); + expect(transaction.attributes).toMatchObject({ + "paperclip.execution.catalogEntryCount": 20, + "paperclip.execution.connectionCount": 2, + }); + for (const name of ["catalog_entries", "matched_connections"]) { + const query = spans.find((span) => span.name === `${transaction.name}.${name}`); + expect(query?.parentSpanContext?.spanId).toBe(transaction.spanContext().spanId); + } + for (const name of ["availability", "descriptors"]) { + const phase = spans.find((span) => span.name === `${prefix}.${name}`); + expect(phase?.parentSpanContext?.spanId).toBe(connectionPhase.spanContext().spanId); + } + } finally { + vi.unstubAllEnvs(); + trace.disable(); + context.disable(); + manager.disable(); + await provider.shutdown(); + } + }); + it("never selects the whole run snapshot or result when it decides access", async () => { // Without agent or company bindings, the project binding is the narrowest match. const fixture = await createListingFixture(db, 3, { broadBindings: false }); diff --git a/server/src/services/execution-preparation-timing.ts b/server/src/services/execution-preparation-timing.ts index f031cd3b143..33f09ee7207 100644 --- a/server/src/services/execution-preparation-timing.ts +++ b/server/src/services/execution-preparation-timing.ts @@ -2,7 +2,7 @@ import { AsyncLocalStorage } from "node:async_hooks"; import { withDatabaseOperation } from "@paperclipai/db"; import { EXECUTION_PREPARATION_SPAN_ATTRS, getActiveStepContext, runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; import { createHash } from "node:crypto"; -import { isSpanContextValid, ROOT_CONTEXT, SpanStatusCode, trace, type Context } from "@opentelemetry/api"; +import { isSpanContextValid, ROOT_CONTEXT, SpanStatusCode, trace, type Context, type Span } from "@opentelemetry/api"; type PreparationPhase = "environment_selection" | "execution_configuration" | "session" | "agent_read" | "chat_completion" | "conversation" | "runtime_state" | "issue_context" @@ -102,6 +102,9 @@ export type ExecutionPhase = | "connectors.browser_use.connection" | "connectors.browser_use.catalog" | "connectors.browser_use.grant" | "connectors.discovery.scheduled" | "connectors.discovery.queue" | "connectors.discovery.build" | "connectors.discovery.agent" | "connectors.discovery.connections" | "connectors.discovery.github" | "connectors.discovery.slack" | "connectors.discovery.speko" + | "connectors.discovery.connections.transaction" | "connectors.discovery.connections.transaction.catalog_entries" + | "connectors.discovery.connections.transaction.matched_connections" + | "connectors.discovery.connections.availability" | "connectors.discovery.connections.descriptors" | "connectors.discovery.policy" | "connectors.discovery.on_demand_policy" | "connectors.slack.candidates" | "connectors.slack.authority" | "connectors.slack.profile" | "connectors.slack.settings" | "connectors.slack.identity" | "connectors.slack.credentials" | "connectors.slack.revision" @@ -124,7 +127,7 @@ export type ExecutionPhase = // recordRunId: the run whose preparation log receives nested measureRunPhase // steps. A traced step opens its own scope, so it carries its preparation // ancestor's target; other scopes (execution attempts) never record. -type RunScope = { runId: string; phase: string; parent?: Context; ended: boolean; details: Map; recordRunId?: string }; +type RunScope = { runId: string; phase: string; parent?: Context; span?: Span; ended: boolean; details: Map; recordRunId?: string }; const ownership = new AsyncLocalStorage(); const endpointEnabled = () => Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()); @@ -148,6 +151,20 @@ export function recordRepositoryObjectCacheDetail(detail: { } catch {} } +export function recordConnectionCatalogCounts(catalogEntryCount: number, connectionCount: number): void { + const owned = ownership.getStore(); + if (!endpointEnabled() || owned?.phase !== "connectors.discovery.connections.transaction" || !owned.span || owned.ended) return; + try { + const span = owned.span; + if (Number.isSafeInteger(catalogEntryCount) && catalogEntryCount >= 0) { + span.setAttribute(EXECUTION_PREPARATION_SPAN_ATTRS.catalogEntryCount, catalogEntryCount); + } + if (Number.isSafeInteger(connectionCount) && connectionCount >= 0) { + span.setAttribute(EXECUTION_PREPARATION_SPAN_ATTRS.connectionCount, connectionCount); + } + } catch {} +} + export function getRunExecutionTraceContext(runId: string): Context | undefined { const owned = ownership.getStore(); return endpointEnabled() && owned?.runId === runId && !owned.ended ? owned.parent : undefined; @@ -209,10 +226,10 @@ function measure(runId: string, name: string, work: () => T | Promise, ind function measureTraced(runId: string, name: string, work: () => T | Promise, independent = false, counts: { fileCount?: number; bytes?: number } = {}, activeParent?: Context, recordRunId?: string): Promise { let operation: Promise | undefined; - const run = (span?: { setStatus(status: { code: number }): void; end(): void }, parent?: Context) => { + const run = (span?: Span, parent?: Context) => { if (operation) return operation; const previous = ownership.getStore(); - const owned: RunScope = { runId, phase: name, + const owned: RunScope = { runId, phase: name, span, parent: parent ?? (!independent && previous?.runId === runId && !previous.ended ? previous.parent : undefined), ended: false, details: !independent && previous?.runId === runId ? previous.details : new Map(), diff --git a/server/src/services/tool-gateway.ts b/server/src/services/tool-gateway.ts index af153b9703d..16bd80e9d73 100644 --- a/server/src/services/tool-gateway.ts +++ b/server/src/services/tool-gateway.ts @@ -31,7 +31,7 @@ import type { BudgetServiceHooks } from "./budgets.js"; import type { PromptInjectionDetectorMetadata } from "@paperclipai/shared"; import { connectionManagementService, type ConnectionManagementContext } from "./connection-management.js"; import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; -import { measureToolDiscovery, measureRunPhase } from "./execution-preparation-timing.js"; +import { measureToolDiscovery, measureRunPhase, recordConnectionCatalogCounts } from "./execution-preparation-timing.js"; import { nativeProviderCatalogFor } from "./native-provider-catalogs.js"; import { callGoogleWorkspaceRest, googleWorkspaceRestAuditEndpoint } from "./google-workspace-rest-dispatch.js"; import { composeConnectionInstructions } from "./connection-instructions.js"; @@ -1837,9 +1837,9 @@ export function createToolGatewayService( // every tool, and one connection can expose hundreds of tools. Both reads // share one read-only snapshot, so they agree like a single join, and the // second read gets only the connections that have listed tools. - const { connectionRows, catalogRows } = await db.transaction( + const readCatalog = () => db.transaction( async (tx) => { - const catalogRows = await tx + const catalogRows = await measureRunPhase("connectors.discovery.connections.transaction.catalog_entries", async () => await tx .select({ catalogEntry: toolCatalogEntries }) .from(toolCatalogEntries) .innerJoin( @@ -1860,7 +1860,7 @@ export function createToolGatewayService( selectedConnectionIds ? inArray(toolConnections.id, selectedConnectionIds) : undefined, ), ) - .orderBy(toolConnections.name, toolCatalogEntries.name); + .orderBy(toolConnections.name, toolCatalogEntries.name)); const connectionIds = [ ...new Set( catalogRows.map(({ catalogEntry }) => catalogEntry.connectionId), @@ -1868,7 +1868,7 @@ export function createToolGatewayService( ]; if (connectionIds.length === 0) return { connectionRows: [], catalogRows }; - const connectionRows = await tx + const connectionRows = await measureRunPhase("connectors.discovery.connections.transaction.matched_connections", async () => await tx .select({ connection: toolConnections, application: toolApplications, @@ -1883,44 +1883,59 @@ export function createToolGatewayService( inArray(toolConnections.id, connectionIds), connectedMcpConnectionFilter(companyId), ), - ); + )); return { connectionRows, catalogRows }; }, { isolationLevel: "repeatable read", accessMode: "read only" }, ); - const connectionRowsById = new Map( - connectionRows.map((row) => [row.connection.id, row]), - ); - const rows = catalogRows.flatMap(({ catalogEntry }) => { - const connectionRow = connectionRowsById.get(catalogEntry.connectionId); - return connectionRow ? [{ catalogEntry, ...connectionRow }] : []; + const { connectionRows, catalogRows } = await measureRunPhase("connectors.discovery.connections.transaction", async () => { + const rows = await readCatalog(); + recordConnectionCatalogCounts(rows.catalogRows.length, rows.connectionRows.length); + return rows; }); + const eligibleRows = await measureRunPhase("connectors.discovery.connections.availability", async () => { + const connectionRowsById = new Map( + connectionRows.map((row) => [row.connection.id, row]), + ); + const rows = catalogRows.flatMap(({ catalogEntry }) => { + const connectionRow = connectionRowsById.get(catalogEntry.connectionId); + return connectionRow ? [{ catalogEntry, ...connectionRow }] : []; + }); - // A plugin connector's tools reach agents only while its plugin is ready - // and still declares the app. Disabled plugins keep their saved setup. - const pluginApps = await pluginAppCatalogFor(db, connectionRows.map(({ connection }) => connection)); - const featureSettings = connectionRows.some(({ application }) => getAppDefinitionForApplicationKey(application.applicationKey)?.experimentalFeatureKey) - ? await instanceSettingsService(db).getExperimental() : undefined; - const eligibleRows = rows.filter( - ({ catalogEntry, connection, application }) => - pluginApps.isAvailable(connectionSourceSlug(connection)) && - (!featureSettings || isConnectorFeatureEnabled(getAppDefinitionForApplicationKey(application.applicationKey), featureSettings)) && - !isRetiredComposioConnection(connection) && - !(nativeProviderCatalogFor(connection) && !nativeProviderCatalogFor(connection)!.isCurrentEntry(catalogEntry)) && - !(restConnectorFor(connection) && !restConnectorFor(connection)!.isCurrent(catalogEntry)) && - !(isRailwayEndpoint(connection.config.url) && (isRailwayToolBlocked(catalogEntry.toolName) || (normalizeRailwayToolName(catalogEntry.toolName).startsWith(RAILWAY_TOOL_PREFIX) && connection.config.railwayApiStatus !== "available"))) && - (((isBrowserUseConnection(connection) || nativeProviderCatalogFor(connection) || restConnectorFor(connection)) && application.type === "rest_api") || - (connection.transport === "mcp_remote" && - application.type === "mcp_http") || - (connection.transport === "local_stdio" && - application.type === "mcp_stdio") || - // Legacy top-level plugin tools keep their original names and are - // listed by the plugin tool path, gated by this connection's installs. - (connection.transport === "plugin" && - application.type === "paperclip_plugin" && - !pluginConnectionIdentity(connection)?.legacyTools && - pluginCatalogEntryCurrent(connection, catalogEntry))), - ); + // A plugin connector's tools reach agents only while its plugin is ready + // and still declares the app. Disabled plugins keep their saved setup. + const pluginApps = await pluginAppCatalogFor(db, connectionRows.map(({ connection }) => connection)); + const featureSettings = connectionRows.some(({ application }) => getAppDefinitionForApplicationKey(application.applicationKey)?.experimentalFeatureKey) + ? await instanceSettingsService(db).getExperimental() : undefined; + return rows.filter( + ({ catalogEntry, connection, application }) => + pluginApps.isAvailable(connectionSourceSlug(connection)) && + (!featureSettings || isConnectorFeatureEnabled(getAppDefinitionForApplicationKey(application.applicationKey), featureSettings)) && + !isRetiredComposioConnection(connection) && + !(nativeProviderCatalogFor(connection) && !nativeProviderCatalogFor(connection)!.isCurrentEntry(catalogEntry)) && + !(restConnectorFor(connection) && !restConnectorFor(connection)!.isCurrent(catalogEntry)) && + !(isRailwayEndpoint(connection.config.url) && (isRailwayToolBlocked(catalogEntry.toolName) || (normalizeRailwayToolName(catalogEntry.toolName).startsWith(RAILWAY_TOOL_PREFIX) && connection.config.railwayApiStatus !== "available"))) && + (((isBrowserUseConnection(connection) || nativeProviderCatalogFor(connection) || restConnectorFor(connection)) && application.type === "rest_api") || + (connection.transport === "mcp_remote" && + application.type === "mcp_http") || + (connection.transport === "local_stdio" && + application.type === "mcp_stdio") || + // Legacy top-level plugin tools keep their original names and are + // listed by the plugin tool path, gated by this connection's installs. + (connection.transport === "plugin" && + application.type === "paperclip_plugin" && + !pluginConnectionIdentity(connection)?.legacyTools && + pluginCatalogEntryCurrent(connection, catalogEntry))), + ); + }); + return measureRunPhase("connectors.discovery.connections.descriptors", () => buildConnectedCatalogDescriptors(eligibleRows)); + } + + function buildConnectedCatalogDescriptors(eligibleRows: Array<{ + catalogEntry: typeof toolCatalogEntries.$inferSelect; + connection: typeof toolConnections.$inferSelect; + application: typeof toolApplications.$inferSelect; + }>): ToolGatewayDescriptor[] { const baseNames = eligibleRows.map( ({ catalogEntry, connection, application }) => { const toolSlug = slugSegment(catalogEntry.toolName, "tool");