diff --git a/doc/plans/2026-10-21-tool-access-boundaries.md b/doc/plans/2026-10-21-tool-access-boundaries.md index 6362f9453ba..0229029fbd3 100644 --- a/doc/plans/2026-10-21-tool-access-boundaries.md +++ b/doc/plans/2026-10-21-tool-access-boundaries.md @@ -175,3 +175,17 @@ Pages and RFC 8693 exchanges, plus a failing upstream request, and assert that issuance/audit rows contain no token material. The complete public service and HTTP rebinding run passes 495 cases. Server typechecking and module boundaries pass. These fixtures do not claim live third-party qualification. + +### Runtime and activity ownership extraction + +Slot persistence and operator controls share the existing runtime supervisor. +The health reader receives the same live deployment options, clock and trusted +host getter. Activity projections and company-scoped history/usage reads live +separately from the action review queue. The queue retains its existing unsigned +request grace period and invalid-signature cancellation writes. + +The entry point is 14,877 lines / 600,156 bytes. Five destination modules total +1,282 lines, with no module larger than 426 lines. All 256 pre-existing bodies +match normalized syntax. The public service and unsigned review-queue coverage +pass before and after (469 combined cases). Server typechecking and module +boundaries pass. No new supervisor, background loop or runtime cache is created. diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.ts index 54e4185fd2c..db1c0088514 100644 --- a/server/src/services/tool-access.ts +++ b/server/src/services/tool-access.ts @@ -1,3 +1,7 @@ +import { createToolRuntimeHealth } from "./tool-access/runtime/health.js"; +import { createToolRuntimeSlots } from "./tool-access/runtime/slots.js"; +import { createToolActivityService } from "./tool-access/activity/service.js"; +import { createToolActionReviewQueue } from "./tool-access/activity/review-queue.js"; import { createToolAccessAudit } from "./tool-access/audit.js"; import { createConnectionTokenBroker } from "./tool-access/token-broker/service.js"; import { assertConfiguredTokenBrokerEndpointsAllowed } from "./tool-access/token-broker/endpoint-policy.js"; @@ -91,8 +95,8 @@ import { sql, } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { agents, connectionGrantMembers, connectionGrantDelegations, connectionGrants, connectionTokenIssuances, authUsers, companyMemberships, companySecretBindings, companySecrets, principalPermissionGrants, userSecretDefinitions, userSecretDeclarations, heartbeatRuns, issues, issueThreadInteractions, plugins, toolAccessAuditEvents, toolApplications, toolActionRequests, toolCatalogEntries, toolConnectionInstalls, toolConnectionAppSnapshots, toolConnectionAppSyncs, toolConnections, toolOauthStates, toolStdioCommandTemplates, toolCallEvents, toolInvocations, toolPolicies, toolGatewaySessions, toolMcpGateways, toolMcpGatewayTokens, toolProfileBindings, toolProfileEntries, toolProfiles, toolRuntimeMetricCounters, toolRuntimeSlots } from "@paperclipai/db"; -import type { AppDefinition, ConnectionGrantKind, CreateToolApplication, CreateToolConnection, ConnectToolApp, ConnectToolAppResult, CreateToolStdioCommandTemplate, FinishToolApp, FinishToolAppResult, FinalizeOAuthAccess, CreateToolProfileBindingForProfile, CreateToolProfileEntryForProfile, DeploymentExposure, DeploymentMode, ImportMcpJson, McpConnectionCredentialRef, McpJsonImportPreview, ToolApplication, ToolCatalogEntry, ToolCatalogRefreshResult, ToolConnection, ToolConnectionInstall, ToolConnectionInstallSnapshot, ToolConnectionRemovalResult, ToolConnectionRemovalSummary, ToolConnectionHealthCheckResult, ToolConnectionHealthStatus, ToolConnectionAuthKind, ToolConnectionCredentialPolicy, ToolConnectionCredentialSource, ToolConnectionTransport, ToolCredentialSecretRef, ToolOAuthStartResult, ToolAppsAttentionResponse, ToolAppMetadataPreflightResult, ToolActionRequest, ToolActionRequestListItem, ToolActionRequestStatus, ToolConnectionActivityResponse, ToolAppConnectionActionSummary, ToolExampleInstallResult, ToolExampleSmokeCheck, ToolExampleSmokeResult, ToolExampleSummary, ToolCallEvent, ToolInvocation, ToolProfileBinding, ToolProfileEntry, ToolPolicyDecision, ToolPolicy, ToolRiskLevel, ToolRuntimeAlertRecommendation, ToolRuntimeHealthSummary, ToolRunDecision, ToolRunDecisionLookup, ToolRuntimeSlot, ToolStdioCommandTemplate, UpdateToolApplication, UpdateToolConnection, PutToolConnectionInstalls, VercelConnectCredentialReference, VercelConnectGrantReference } from "@paperclipai/shared"; +import { agents, connectionGrantMembers, connectionGrantDelegations, connectionGrants, connectionTokenIssuances, authUsers, companyMemberships, companySecretBindings, companySecrets, principalPermissionGrants, userSecretDefinitions, userSecretDeclarations, issueThreadInteractions, plugins, toolAccessAuditEvents, toolApplications, toolActionRequests, toolCatalogEntries, toolConnectionInstalls, toolConnectionAppSnapshots, toolConnectionAppSyncs, toolConnections, toolOauthStates, toolStdioCommandTemplates, toolCallEvents, toolInvocations, toolPolicies, toolGatewaySessions, toolMcpGateways, toolMcpGatewayTokens, toolProfileBindings, toolProfileEntries, toolProfiles, toolRuntimeSlots } from "@paperclipai/db"; +import type { AppDefinition, ConnectionGrantKind, CreateToolApplication, CreateToolConnection, ConnectToolApp, ConnectToolAppResult, CreateToolStdioCommandTemplate, FinishToolApp, FinishToolAppResult, FinalizeOAuthAccess, CreateToolProfileBindingForProfile, CreateToolProfileEntryForProfile, DeploymentExposure, DeploymentMode, ImportMcpJson, McpConnectionCredentialRef, McpJsonImportPreview, ToolApplication, ToolCatalogEntry, ToolCatalogRefreshResult, ToolConnection, ToolConnectionInstall, ToolConnectionInstallSnapshot, ToolConnectionRemovalResult, ToolConnectionRemovalSummary, ToolConnectionHealthCheckResult, ToolConnectionHealthStatus, ToolConnectionAuthKind, ToolConnectionCredentialPolicy, ToolConnectionCredentialSource, ToolConnectionTransport, ToolCredentialSecretRef, ToolOAuthStartResult, ToolAppsAttentionResponse, ToolAppMetadataPreflightResult, ToolAppConnectionActionSummary, ToolExampleInstallResult, ToolExampleSmokeCheck, ToolExampleSmokeResult, ToolExampleSummary, ToolProfileBinding, ToolProfileEntry, ToolPolicyDecision, ToolPolicy, ToolRiskLevel, ToolStdioCommandTemplate, UpdateToolApplication, UpdateToolConnection, PutToolConnectionInstalls, VercelConnectCredentialReference, VercelConnectGrantReference } from "@paperclipai/shared"; import { CLASS3_STATIC_LEASE_ALLOWLIST, GITHUB_CONNECTOR_PROFILES, @@ -148,19 +152,12 @@ import type { ConfigureRailwaySsh, RailwaySshSetup } from "@paperclipai/shared"; import { generateRailwaySshKey, RAILWAY_SSH_SECRET_PATH, validateRailwayKnownHosts } from "./railway-ssh.js"; import { createRailwayClient, discoverRailwayWorkspace, isRailwayConnection, isRailwayEndpoint, isRailwayToolBlocked, normalizeRailwayToolName, RAILWAY_TOOLS, RAILWAY_TOOL_PREFIX, RailwayError } from "./railway.js"; import { toolAccessPolicyService } from "./tool-access-policy.js"; -import { - readSignedToolArgumentsPayload, - TOOL_ACTION_REQUEST_SIGNING_GRACE_MS, -} from "./tool-content-guards.js"; import { googleChatToolDescription } from "./google-chat-tool-policy.js"; import { gmailArchiveToolDescription } from "./gmail-archive-tool-policy.js"; -import { TOOL_RUNTIME_AUDIT_WRITE_FAILURE_METRIC } from "./tool-runtime-metrics.js"; -import { - createToolRuntimeSupervisor, - ToolRuntimeSupervisorError, -} from "./tool-runtime-supervisor.js"; -import { listConnectionLifecycleEvents } from "./tool-connection-activity.js"; + +import { createToolRuntimeSupervisor } from "./tool-runtime-supervisor.js"; + import { isRetiredComposioConnection, RETIRED_COMPOSIO_MESSAGE } from "@paperclipai/shared"; import { appWithPaperclipCloudConnectorAvailability, @@ -643,26 +640,6 @@ function isToolConnectionForeignKeyViolation(error: unknown): boolean { }); } -function numberValue(value: unknown): number | null { - const parsed = typeof value === "number" ? value : Number(value); - return Number.isFinite(parsed) ? parsed : null; -} - -function percent(numerator: number, denominator: number): number { - if (denominator <= 0) return 0; - return Math.round((numerator / denominator) * 1000) / 10; -} - -function percentile(values: number[], p: number): number | null { - if (values.length === 0) return null; - const sorted = [...values].sort((a, b) => a - b); - const index = Math.min( - sorted.length - 1, - Math.max(0, Math.ceil((p / 100) * sorted.length) - 1), - ); - return sorted[index] ?? null; -} - /** * The key namespace `connectGalleryApp`, `reconnectGalleryApp` and * `createOrRotateOAuthSecret` mint for credentials a connection owns outright. @@ -952,45 +929,6 @@ function toCatalogEntryForConnection( return catalogEntry; } -function toRuntimeSlot( - row: typeof toolRuntimeSlots.$inferSelect, -): ToolRuntimeSlot { - return { - id: row.id, - companyId: row.companyId, - applicationId: row.applicationId, - connectionId: row.connectionId, - projectWorkspaceId: row.projectWorkspaceId, - executionWorkspaceId: row.executionWorkspaceId, - issueId: row.issueId, - ownerScopeType: row.ownerScopeType, - ownerScopeId: row.ownerScopeId, - runtimeKind: row.runtimeKind, - slotKey: row.slotKey, - status: row.status, - reuseKey: row.reuseKey, - workspaceScope: row.workspaceScope, - credentialScopeHash: row.credentialScopeHash, - provider: row.provider, - providerRef: row.providerRef, - processId: row.processId, - commandTemplateKey: row.commandTemplateKey, - healthStatus: row.healthStatus, - healthMessage: row.healthMessage, - lastHealthCheckAt: row.lastHealthCheckAt, - lastStartedAt: row.lastStartedAt, - startedAt: row.startedAt, - stoppedAt: row.stoppedAt, - lastUsedAt: row.lastUsedAt, - idleExpiresAt: row.idleExpiresAt, - idleDeadlineAt: row.idleDeadlineAt, - lastError: row.lastError, - metadata: row.metadata ?? null, - createdAt: row.createdAt, - updatedAt: row.updatedAt, - }; -} - function builtInStdioTemplate( templateId: string, ): ToolStdioCommandTemplate | null { @@ -1049,142 +987,6 @@ function toStdioCommandTemplate( }; } -function toToolInvocation( - row: typeof toolInvocations.$inferSelect, -): ToolInvocation { - return { - id: row.id, - companyId: row.companyId, - idempotencyKey: row.idempotencyKey, - actorType: row.actorType as ToolInvocation["actorType"], - actorId: row.actorId, - agentId: row.agentId, - issueId: row.issueId, - runId: row.runId, - applicationId: row.applicationId, - connectionId: row.connectionId, - catalogEntryId: row.catalogEntryId, - toolName: row.toolName, - argumentsHash: row.argumentsHash, - argumentsSummary: row.argumentsSummary ?? null, - policyDecision: row.policyDecision, - matchedPolicyIds: row.matchedPolicyIds, - approvalState: row.approvalState, - status: row.status, - upstreamRequestId: row.upstreamRequestId, - resultHash: row.resultHash, - resultSummary: row.resultSummary ?? null, - resultSizeBytes: row.resultSizeBytes, - resultArtifactId: row.resultArtifactId, - errorCode: row.errorCode, - errorMessage: row.errorMessage, - startedAt: row.startedAt, - completedAt: row.completedAt, - createdAt: row.createdAt, - updatedAt: row.updatedAt, - }; -} - -function toToolActionRequest( - row: typeof toolActionRequests.$inferSelect, -): ToolActionRequest { - return { - id: row.id, - companyId: row.companyId, - invocationId: row.invocationId, - issueId: row.issueId, - interactionId: row.interactionId, - approvalId: row.approvalId, - status: row.status, - canonicalArgumentsHash: row.canonicalArgumentsHash, - canonicalArgumentsSummary: row.canonicalArgumentsSummary, - signedArguments: row.signedArguments, - previewMarkdown: row.previewMarkdown, - requestedByAgentId: row.requestedByAgentId, - requestedByUserId: row.requestedByUserId, - resolvedByAgentId: row.resolvedByAgentId, - resolvedByUserId: row.resolvedByUserId, - decidedByAgentId: row.decidedByAgentId, - decidedByUserId: row.decidedByUserId, - decidedAt: row.decidedAt, - expiresAt: row.expiresAt, - resolvedAt: row.resolvedAt, - createdAt: row.createdAt, - updatedAt: row.updatedAt, - }; -} - -function toToolCallEvent( - row: typeof toolCallEvents.$inferSelect, -): ToolCallEvent { - return { - id: row.id, - companyId: row.companyId, - eventType: row.eventType, - actorType: row.actorType as ToolCallEvent["actorType"], - actorId: row.actorId, - agentId: row.agentId, - runId: row.runId, - issueId: row.issueId, - applicationId: row.applicationId, - connectionId: row.connectionId, - catalogEntryId: row.catalogEntryId, - invocationId: row.invocationId, - actionRequestId: row.actionRequestId, - runtimeSlotId: row.runtimeSlotId, - toolName: row.toolName, - decision: row.decision, - matchedPolicyIds: row.matchedPolicyIds, - reasonCode: row.reasonCode, - outcome: row.outcome, - latencyMs: row.latencyMs, - argumentsSummary: row.argumentsSummary ?? null, - requestHash: row.requestHash, - requestSummary: row.requestSummary ?? null, - resultHash: row.resultHash, - resultSummary: row.resultSummary ?? null, - resultSizeBytes: row.resultSizeBytes, - redactionPlan: row.redactionPlan ?? null, - rateLimitState: row.rateLimitState ?? null, - metadata: row.metadata ?? null, - errorCode: row.errorCode, - errorMessage: row.errorMessage, - createdAt: row.createdAt, - }; -} - -function userFallbackName(userId: string): string { - if (userId === "local-board") return "Board"; - return userId; -} - -function denialReasonForDecision( - invocation: typeof toolInvocations.$inferSelect, - latestAuditEvent: typeof toolCallEvents.$inferSelect | null, -) { - if ( - invocation.status === "denied" || - invocation.status === "rate_limited" || - invocation.status === "failed" || - invocation.status === "timed_out" - ) { - return ( - invocation.errorMessage ?? - invocation.errorCode ?? - latestAuditEvent?.reasonCode ?? - null - ); - } - if ( - latestAuditEvent?.outcome === "denied" || - latestAuditEvent?.outcome === "failure" || - latestAuditEvent?.outcome === "timeout" - ) { - return latestAuditEvent.errorMessage ?? latestAuditEvent.reasonCode ?? null; - } - return null; -} - function toPolicy(row: typeof toolPolicies.$inferSelect): ToolPolicy { return { id: row.id, @@ -1809,6 +1611,11 @@ export function toolAccessService( ? createVercelConnectClient() : options.vercelConnectClient; const runtimeSupervisor = createToolRuntimeSupervisor(db, options); + const runtimeSlots = createToolRuntimeSlots(db, runtimeSupervisor); + const { ensureRuntimeSlot } = runtimeSlots; + const runtimeHealth = createToolRuntimeHealth(db, options, now, trustedRuntimeHost); + const toolActivity = createToolActivityService(db, getConnectionRow); + const toolActionReviewQueue = createToolActionReviewQueue(db); // These maps remove duplicate work inside one service instance. OAuth also // uses the database refresh lease below as its cross-process boundary. const oauthRefreshFlights = new Map>(); @@ -2509,466 +2316,6 @@ export function toolAccessService( return syncAggregatorApps(connectionId, true, actor); } - function runtimeAlert( - input: ToolRuntimeAlertRecommendation, - ): ToolRuntimeAlertRecommendation { - return input; - } - - function buildRuntimeAlerts(input: { - stuckStartingSlots: number; - stuckRunningSlots: number; - timeoutRate: number; - timeoutCount: number; - failureRate: number; - failureCount: number; - capacityDeferrals: number; - restartAttempts: number; - restartSuppressions: number; - degradedConnections: number; - disabledConnections: number; - missingSecretFailures: number; - auditWriteFailures: number; - }): ToolRuntimeAlertRecommendation[] { - const runbookSection = "doc/MCP-RUNTIME-OPERATIONS.md"; - const timeoutSeverity = - input.timeoutCount >= 10 || input.timeoutRate >= 25 - ? "critical" - : input.timeoutCount >= 3 && input.timeoutRate >= 10 - ? "warning" - : "warning"; - const failureSeverity = - input.failureCount >= 10 || input.failureRate >= 25 - ? "critical" - : input.failureCount >= 5 && input.failureRate >= 10 - ? "warning" - : "warning"; - const restartSeverity = - input.restartSuppressions > 0 ? "critical" : "warning"; - return [ - runtimeAlert({ - name: "mcp_runtime_stuck_starting_slot", - severity: "critical", - status: input.stuckStartingSlots > 0 ? "firing" : "ok", - threshold: "Any starting slot older than 5 minutes.", - observed: `${input.stuckStartingSlots} stuck starting slot(s).`, - description: - "A local stdio runtime slot is stuck before it reaches running state.", - firstResponderAction: - "Inspect the slot health/logs, stop the slot, restart it once, then disable the connection if the slot sticks again.", - runbookSection, - }), - runtimeAlert({ - name: "mcp_runtime_stuck_running_slot", - severity: "critical", - status: input.stuckRunningSlots > 0 ? "firing" : "ok", - threshold: "Any running slot with no progress for 5 minutes.", - observed: `${input.stuckRunningSlots} stuck running slot(s).`, - description: - "A runtime slot is running but has not recorded progress inside the supervisor stuck-slot window.", - firstResponderAction: - "Inspect recent audit events and active tool calls; restart the slot only after confirming no healthy call is still in progress.", - runbookSection, - }), - runtimeAlert({ - name: "mcp_runtime_high_timeout_rate", - severity: timeoutSeverity, - status: - input.timeoutCount >= 3 && input.timeoutRate >= 10 ? "firing" : "ok", - threshold: - "Warning at >=3 timeouts and >=10% timeout rate in 1 hour; critical at >=10 timeouts or >=25%.", - observed: `${input.timeoutCount} timeout(s), ${input.timeoutRate}% timeout rate.`, - description: - "Tool gateway calls are timing out or being runtime-deferred at an elevated rate.", - firstResponderAction: - "Check upstream MCP health, Paperclip runtime capacity, and recent gateway audit failures before retrying workloads.", - runbookSection, - }), - runtimeAlert({ - name: "mcp_runtime_high_error_rate", - severity: failureSeverity, - status: - input.failureCount >= 5 && input.failureRate >= 10 ? "firing" : "ok", - threshold: - "Warning at >=5 failures and >=10% failure rate in 1 hour; critical at >=10 failures or >=25%.", - observed: `${input.failureCount} failure(s), ${input.failureRate}% failure rate.`, - description: - "Tool gateway calls are failing after policy authorization.", - firstResponderAction: - "Group audit failures by reasonCode, then fix credentials/config or disable the affected connection.", - runbookSection, - }), - runtimeAlert({ - name: "mcp_runtime_capacity_deferrals_repeated", - severity: input.capacityDeferrals >= 10 ? "critical" : "warning", - status: input.capacityDeferrals >= 3 ? "firing" : "ok", - threshold: - "Warning at >=3 capacity deferrals in 1 hour; critical at >=10.", - observed: `${input.capacityDeferrals} capacity deferral(s) in 1 hour.`, - description: - "The runtime supervisor is refusing local stdio work because company or host slot capacity is exhausted.", - firstResponderAction: - "Stop idle/stale slots, lower noisy workloads, or raise slot caps only after confirming host capacity.", - runbookSection, - }), - runtimeAlert({ - name: "mcp_runtime_restart_storm", - severity: restartSeverity, - status: - input.restartSuppressions > 0 || input.restartAttempts >= 3 - ? "firing" - : "ok", - threshold: - "Warning at >=3 restarts in 1 hour; critical on any restart suppression.", - observed: `${input.restartAttempts} restart attempt(s), ${input.restartSuppressions} suppression(s).`, - description: - "Runtime slots are restarting repeatedly or have hit restart-storm suppression.", - firstResponderAction: - "Stop the affected slot, inspect stderr/audit reason codes, and keep the connection disabled until the template/upstream is fixed.", - runbookSection, - }), - runtimeAlert({ - name: "mcp_runtime_connection_health_degraded", - severity: input.degradedConnections > 0 ? "critical" : "warning", - status: - input.degradedConnections > 0 || input.disabledConnections > 0 - ? "firing" - : "ok", - threshold: - "Any active enabled connection with degraded/failed/missing-secret health, or any disabled enabled-path connection.", - observed: `${input.degradedConnections} degraded connection(s), ${input.disabledConnections} disabled connection(s).`, - description: - "A configured MCP connection is not healthy or has been disabled.", - firstResponderAction: - "Run a connection health check, refresh catalog after recovery, or keep the connection disabled and route agents to alternatives.", - runbookSection, - }), - runtimeAlert({ - name: "mcp_runtime_missing_secret_failures", - severity: input.missingSecretFailures >= 3 ? "critical" : "warning", - status: input.missingSecretFailures > 0 ? "firing" : "ok", - threshold: - "Warning on any missing-secret failure; critical at >=3 in 1 hour.", - observed: `${input.missingSecretFailures} missing-secret failure(s) in 1 hour.`, - description: - "A connection or tool call needed a bound secret that could not be resolved.", - firstResponderAction: - "Check secret bindings and provider health without printing secret values; rotate or rebind missing secrets.", - runbookSection, - }), - runtimeAlert({ - name: "mcp_runtime_audit_write_failures", - severity: "critical", - status: input.auditWriteFailures > 0 ? "firing" : "ok", - threshold: "Any audit write failure.", - observed: `${input.auditWriteFailures} audit write failure(s) in 1 hour.`, - description: - "Tool gateway audit writes failed, reducing incident traceability.", - firstResponderAction: - "Treat as a control-plane incident: check database writes, activity log writes, and retry only after audit durability is restored.", - runbookSection, - }), - ]; - } - - async function runtimeHealth( - companyId: string, - ): Promise { - const generatedAt = now(); - const windowStartedAt = new Date(generatedAt.getTime() - 60 * 60 * 1000); - const stuckSlotMs = 5 * 60 * 1000; - const [ - slots, - connections, - auditRows, - callEvents, - auditWriteFailureCounterRows, - ] = await Promise.all([ - db - .select() - .from(toolRuntimeSlots) - .where(eq(toolRuntimeSlots.companyId, companyId)), - db - .select() - .from(toolConnections) - .where(eq(toolConnections.companyId, companyId)), - db - .select() - .from(toolAccessAuditEvents) - .where( - and( - eq(toolAccessAuditEvents.companyId, companyId), - gte(toolAccessAuditEvents.createdAt, windowStartedAt), - ), - ) - .orderBy(desc(toolAccessAuditEvents.createdAt)), - db - .select() - .from(toolCallEvents) - .where( - and( - eq(toolCallEvents.companyId, companyId), - gte(toolCallEvents.createdAt, windowStartedAt), - ), - ) - .orderBy(desc(toolCallEvents.createdAt)), - db - .select({ - count: sql`coalesce(sum(${toolRuntimeMetricCounters.count}), 0)::int`, - }) - .from(toolRuntimeMetricCounters) - .where( - and( - eq(toolRuntimeMetricCounters.companyId, companyId), - eq( - toolRuntimeMetricCounters.metric, - TOOL_RUNTIME_AUDIT_WRITE_FAILURE_METRIC, - ), - gte(toolRuntimeMetricCounters.bucketStartAt, windowStartedAt), - ), - ), - ]); - const activeSlots = slots.filter( - (slot) => - slot.status === "starting" || - slot.status === "running" || - slot.status === "idle", - ); - const staleActiveSlots = activeSlots.filter((slot) => { - const lastProgressAt = - slot.lastUsedAt ?? slot.startedAt ?? slot.updatedAt; - return generatedAt.getTime() - lastProgressAt.getTime() > stuckSlotMs; - }); - const callTerminalEvents = callEvents.filter( - (event) => - event.eventType === "call_completed" || - event.eventType === "call_failed" || - event.eventType === "call_denied", - ); - const toolCallsLastHour = callTerminalEvents.length; - const toolTimeoutsLastHour = callTerminalEvents.filter( - (event) => event.outcome === "timeout", - ).length; - const toolFailuresLastHour = callTerminalEvents.filter( - (event) => event.outcome === "failure", - ).length; - const durations = auditRows - .map((row) => numberValue(asRecord(row.details).durationMs)) - .filter((value): value is number => value !== null && value >= 0); - const capacityDeferrals = auditRows.filter( - (row) => - row.action === "runtime_deferred" || - row.reasonCode === "runtime_company_capacity_exhausted" || - row.reasonCode === "runtime_host_capacity_exhausted", - ).length; - const restartAttempts = auditRows.filter( - (row) => - row.action === "runtime_started" && row.reasonCode !== "lazy_start", - ).length; - const restartSuppressions = auditRows.filter( - (row) => - row.action === "runtime_restart_suppressed" || - row.reasonCode === "runtime_restart_suppressed", - ).length; - const idleEvictions = auditRows.filter( - (row) => - row.action === "runtime_stopped" && - row.reasonCode === "idle_ttl_expired", - ).length; - const missingSecretFailures = auditRows.filter( - (row) => - row.reasonCode === "missing_secret" || - (row.outcome === "failure" && row.reasonCode?.includes("secret")), - ).length; - const legacyAuditWriteFailures = auditRows.filter( - (row) => - row.action === "runtime_audit_write_failed" || - row.reasonCode === "audit_write_failed", - ).length; - const auditWriteFailuresMetric = - Number(auditWriteFailureCounterRows[0]?.count ?? 0) + - legacyAuditWriteFailures; - const enabledPathConnections = connections.filter( - (connection) => connection.status === "active" && connection.enabled, - ); - const activeConnections = enabledPathConnections.length; - const disabledConnections = connections.filter( - (connection) => connection.status === "disabled", - ).length; - const degradedConnections = enabledPathConnections.filter((connection) => - ["degraded", "failed", "error", "missing_secret"].includes( - connection.healthStatus, - ), - ).length; - const metrics = { - windowStartedAt, - windowEndedAt: generatedAt, - activeSlots: activeSlots.length, - startingSlots: slots.filter((slot) => slot.status === "starting").length, - runningSlots: slots.filter((slot) => slot.status === "running").length, - idleSlots: slots.filter((slot) => slot.status === "idle").length, - failedSlots: slots.filter( - (slot) => slot.status === "failed" || slot.status === "error", - ).length, - stoppedSlots: slots.filter( - (slot) => slot.status === "stopped" || slot.status === "disabled", - ).length, - stuckStartingSlots: staleActiveSlots.filter( - (slot) => slot.status === "starting", - ).length, - stuckRunningSlots: staleActiveSlots.filter( - (slot) => slot.status === "running", - ).length, - capacityDeferralsLastHour: capacityDeferrals, - restartAttemptsLastHour: restartAttempts, - restartSuppressionsLastHour: restartSuppressions, - idleEvictionsLastHour: idleEvictions, - toolCallsLastHour, - toolTimeoutsLastHour, - toolFailuresLastHour, - timeoutRateLastHour: percent(toolTimeoutsLastHour, toolCallsLastHour), - failureRateLastHour: percent(toolFailuresLastHour, toolCallsLastHour), - averageToolLatencyMsLastHour: - durations.length > 0 - ? Math.round( - durations.reduce((sum, value) => sum + value, 0) / - durations.length, - ) - : null, - p95ToolLatencyMsLastHour: percentile(durations, 95), - missingSecretFailuresLastHour: missingSecretFailures, - auditWriteFailuresLastHour: auditWriteFailuresMetric, - activeConnections, - disabledConnections, - degradedConnections, - remoteHttpConnections: connections.filter( - (connection) => - connection.status !== "archived" && - connection.transport === "mcp_remote", - ).length, - localStdioConnections: connections.filter( - (connection) => - connection.status !== "archived" && - connection.transport === "local_stdio", - ).length, - }; - const recommendations = buildRuntimeAlerts({ - stuckStartingSlots: metrics.stuckStartingSlots, - stuckRunningSlots: metrics.stuckRunningSlots, - timeoutRate: metrics.timeoutRateLastHour, - timeoutCount: metrics.toolTimeoutsLastHour, - failureRate: metrics.failureRateLastHour, - failureCount: metrics.toolFailuresLastHour, - capacityDeferrals, - restartAttempts, - restartSuppressions, - degradedConnections, - disabledConnections, - missingSecretFailures, - auditWriteFailures: metrics.auditWriteFailuresLastHour, - }); - const firing = recommendations.filter((alert) => alert.status === "firing"); - const status = firing.some((alert) => alert.severity === "critical") - ? "critical" - : firing.length > 0 - ? "degraded" - : "ok"; - const deploymentMode = options.deploymentMode ?? "local_trusted"; - const deploymentExposure = options.deploymentExposure ?? "private"; - const localStdioSupported = - deploymentMode === "local_trusted" || Boolean(trustedRuntimeHost()); - return { - status, - generatedAt, - runbookPath: "doc/MCP-RUNTIME-OPERATIONS.md", - metrics, - supportMatrix: { - remoteHttp: { - supported: true, - note: "mcp_remote MCP connections are supported in hosted cloud and local deployments.", - }, - localStdio: { - supported: localStdioSupported, - note: localStdioSupported - ? "local_stdio is available for local trusted mode or through the configured trusted MCP runtime host." - : `local_stdio should stay disabled for ${deploymentMode}/${deploymentExposure}; use mcp_remote or configure a trusted runtime worker.`, - }, - }, - alerts: firing, - recommendations, - }; - } - - async function runtimeSlotById( - companyId: string, - slotId: string, - ): Promise { - const [row] = await db - .select() - .from(toolRuntimeSlots) - .where( - and( - eq(toolRuntimeSlots.companyId, companyId), - eq(toolRuntimeSlots.id, slotId), - ), - ) - .limit(1); - if (!row) throw notFound("Runtime slot not found"); - return toRuntimeSlot(row); - } - - function runtimeSupervisorHttpError(error: ToolRuntimeSupervisorError) { - return new HttpError(error.status, error.message, { - code: error.reasonCode, - ...error.details, - }); - } - - async function controlRuntimeSlot(input: { - companyId: string; - slotId: string; - action: "stop" | "restart"; - actor?: ActorInfo; - }): Promise { - try { - if (input.action === "stop") { - await runtimeSupervisor.stopSlot({ - companyId: input.companyId, - slotId: input.slotId, - reason: "operator_stop", - }); - } else { - await runtimeSupervisor.restartSlot({ - companyId: input.companyId, - slotId: input.slotId, - }); - } - const slot = await runtimeSlotById(input.companyId, input.slotId); - await logActivity(db, { - companyId: input.companyId, - actorType: input.actor?.actorType ?? "system", - actorId: input.actor?.actorId ?? "tool-access-service", - action: - input.action === "stop" - ? "tool_runtime_slot.operator_stopped" - : "tool_runtime_slot.operator_restarted", - entityType: "tool_runtime_slot", - entityId: input.slotId, - details: { - runtimeKind: slot.runtimeKind, - status: slot.status, - slotKey: slot.slotKey, - }, - }); - return slot; - } catch (error) { - if (error instanceof ToolRuntimeSupervisorError) { - throw runtimeSupervisorHttpError(error); - } - throw error; - } - } - async function assertApplication(companyId: string, applicationId: string) { const [row] = await db .select() @@ -4110,42 +3457,6 @@ export function toolAccessService( }; } - async function ensureRuntimeSlot( - connection: typeof toolConnections.$inferSelect, - ): Promise { - if (connection.transport !== "local_stdio" || connection.config.templateId === "paperclip.cognee-cloud") return null; - const slotKey = `mcp:${connection.companyId}:${connection.id}`; - const [existing] = await db - .select() - .from(toolRuntimeSlots) - .where( - and( - eq(toolRuntimeSlots.companyId, connection.companyId), - eq(toolRuntimeSlots.slotKey, slotKey), - ), - ); - if (existing) return toRuntimeSlot(existing); - const [created] = await db - .insert(toolRuntimeSlots) - .values({ - companyId: connection.companyId, - applicationId: connection.applicationId, - connectionId: connection.id, - slotKey, - ownerScopeType: "connection", - ownerScopeId: connection.id, - runtimeKind: "local_stdio", - status: "stopped", - provider: "paperclip", - providerRef: `template:${String(connection.config.templateId)}`, - commandTemplateKey: String(connection.config.templateId), - healthStatus: "unchecked", - metadata: { templateId: connection.config.templateId }, - }) - .returning(); - return toRuntimeSlot(created); - } - async function vaultGrantForConnection( connection: typeof toolConnections.$inferSelect, actor?: ActorInfo, @@ -15030,84 +14341,7 @@ export function toolAccessService( }); return toConnectionGrant(grant); }, - - getConnectionUsage: async ( - idOrUid: string, - range: "7d" | "30d", - companyId?: string, - ) => { - const connection = await getConnectionRow(idOrUid, companyId); - const days = range === "30d" ? 30 : 7; - const start = new Date(); - start.setUTCHours(0, 0, 0, 0); - start.setUTCDate(start.getUTCDate() - days + 1); - const [issuances, invocations] = await Promise.all([ - db - .select({ - createdAt: connectionTokenIssuances.createdAt, - outcome: connectionTokenIssuances.outcome, - path: connectionTokenIssuances.path, - }) - .from(connectionTokenIssuances) - .where( - and( - eq(connectionTokenIssuances.companyId, connection.companyId), - eq(connectionTokenIssuances.connectionId, connection.id), - gte(connectionTokenIssuances.createdAt, start), - ), - ), - db - .select({ - createdAt: toolInvocations.createdAt, - riskLevel: toolInvocations.riskLevel, - }) - .from(toolInvocations) - .where( - and( - eq(toolInvocations.companyId, connection.companyId), - eq(toolInvocations.connectionId, connection.id), - gte(toolInvocations.createdAt, start), - ), - ), - ]); - const buckets = Array.from({ length: days }, (_, offset) => { - const date = new Date(start); - date.setUTCDate(start.getUTCDate() + offset); - return { - date: date.toISOString().slice(0, 10), - issuances: { - total: 0, - byOutcome: {} as Record, - byPath: {} as Record, - }, - invocations: { total: 0, byRiskLevel: {} as Record }, - deliveries: { received: 0, forwarded: 0 }, - }; - }); - const byDate = new Map(buckets.map((bucket) => [bucket.date, bucket])); - for (const row of issuances) { - const bucket = byDate.get(row.createdAt.toISOString().slice(0, 10)); - if (!bucket) continue; - bucket.issuances.total += 1; - bucket.issuances.byOutcome[row.outcome] = - (bucket.issuances.byOutcome[row.outcome] ?? 0) + 1; - bucket.issuances.byPath[row.path] = - (bucket.issuances.byPath[row.path] ?? 0) + 1; - } - for (const row of invocations) { - const bucket = byDate.get(row.createdAt.toISOString().slice(0, 10)); - if (!bucket) continue; - const riskLevel = row.riskLevel ?? "unknown"; - bucket.invocations.total += 1; - bucket.invocations.byRiskLevel[riskLevel] = - (bucket.invocations.byRiskLevel[riskLevel] ?? 0) + 1; - } - return { - connection: { id: connection.id, uid: connection.uid }, - range, - buckets, - }; - }, + ...toolActivity, listConnectionInstalls, @@ -15484,471 +14718,14 @@ export function toolAccessService( .filter((row) => !nativeDaytona || (row.status !== "removed" && isDaytonaToolName(row.toolName))) .map((row) => toCatalogEntryForConnection(row, connection)); }, - - /** Recent tool-call events for one connection — drives App detail · Recent activity. */ - listConnectionActivity: async ( - connectionId: string, - companyId?: string, - limit = 20, - ): Promise => { - const connection = await getConnectionRow(connectionId, companyId); - const safeLimit = Math.max(1, Math.min(100, Math.floor(limit))); - const rows = await db - .select() - .from(toolCallEvents) - .where( - and( - eq(toolCallEvents.companyId, connection.companyId), - eq(toolCallEvents.connectionId, connection.id), - ), - ) - .orderBy(desc(toolCallEvents.createdAt)) - .limit(safeLimit); - const events = rows.map(toToolCallEvent); - - const issueIds = [ - ...new Set(rows.map((row) => row.issueId).filter(Boolean)), - ] as string[]; - const issueRows = issueIds.length - ? await db - .select({ - id: issues.id, - identifier: issues.identifier, - title: issues.title, - }) - .from(issues) - .where( - and( - eq(issues.companyId, connection.companyId), - inArray(issues.id, issueIds), - ), - ) - : []; - const issueMap = Object.fromEntries( - issueRows.map((issue) => [ - issue.id, - { - identifier: issue.identifier ?? issue.id, - title: issue.title, - }, - ]), - ); - - const actionRequestIds = [ - ...new Set(rows.map((row) => row.actionRequestId).filter(Boolean)), - ] as string[]; - const requestRows = actionRequestIds.length - ? await db - .select({ - id: toolActionRequests.id, - status: toolActionRequests.status, - resolvedByAgentId: toolActionRequests.resolvedByAgentId, - resolvedByUserId: toolActionRequests.resolvedByUserId, - }) - .from(toolActionRequests) - .where( - and( - eq(toolActionRequests.companyId, connection.companyId), - inArray(toolActionRequests.id, actionRequestIds), - ), - ) - : []; - - const resolverAgentIds = [ - ...new Set( - requestRows.map((row) => row.resolvedByAgentId).filter(Boolean), - ), - ] as string[]; - const resolverUserIds = [ - ...new Set( - requestRows.map((row) => row.resolvedByUserId).filter(Boolean), - ), - ] as string[]; - const resolverAgents = resolverAgentIds.length - ? await db - .select({ id: agents.id, name: agents.name }) - .from(agents) - .where( - and( - eq(agents.companyId, connection.companyId), - inArray(agents.id, resolverAgentIds), - ), - ) - : []; - const resolverUsers = resolverUserIds.length - ? await db - .select({ - id: authUsers.id, - name: authUsers.name, - email: authUsers.email, - }) - .from(authUsers) - .where(inArray(authUsers.id, resolverUserIds)) - : []; - const resolverAgentNames = new Map( - resolverAgents.map((agent) => [agent.id, agent.name]), - ); - const resolverUserNames = new Map( - resolverUsers.map((user) => [ - user.id, - user.name?.trim() || user.email?.trim() || user.id, - ]), - ); - const actionRequestMap = Object.fromEntries( - requestRows.map((request) => [ - request.id, - { - status: request.status, - resolverDisplayName: request.resolvedByAgentId - ? (resolverAgentNames.get(request.resolvedByAgentId) ?? - request.resolvedByAgentId) - : request.resolvedByUserId - ? (resolverUserNames.get(request.resolvedByUserId) ?? - userFallbackName(request.resolvedByUserId)) - : null, - resolvedByAgentId: request.resolvedByAgentId, - resolvedByUserId: request.resolvedByUserId, - }, - ]), - ); - - const lifecycleEvents = await listConnectionLifecycleEvents(db, { - companyId: connection.companyId, - connectionIds: [connection.id], - limit: safeLimit, - }); - - return { - connectionId: connection.id, - events, - lifecycleEvents, - issues: issueMap, - actionRequests: actionRequestMap, - }; - }, - - /** - * List "Ask first" action requests for the review queue, enriched with the - * connection/app context the prosumer card renders. Defaults to pending. - */ - listActionRequests: async ( - companyId: string, - status: ToolActionRequestStatus = "pending", - ): Promise => { - const requests = await db - .select() - .from(toolActionRequests) - .where( - and( - eq(toolActionRequests.companyId, companyId), - eq(toolActionRequests.status, status), - ), - ) - .orderBy(desc(toolActionRequests.createdAt)); - if (requests.length === 0) return []; - - const invocationIds = [ - ...new Set(requests.map((request) => request.invocationId)), - ]; - const invocations = await db - .select() - .from(toolInvocations) - .where( - and( - eq(toolInvocations.companyId, companyId), - inArray(toolInvocations.id, invocationIds), - ), - ); - const invocationById = new Map( - invocations.map((invocation) => [invocation.id, invocation]), - ); - let visibleRequests = requests; - if (status === "pending") { - // A pending request that the creator has not signed yet is still being - // set up. The gateway creates the row (signedArguments = null) and signs - // it in a second step, so a review-queue read can observe the row inside - // that window. Hide such a request from the queue, but do not cancel it — - // cancelling here races the two-step create and makes the later approve - // fail with action_not_pending. Only cancel a request that carries a - // signature we cannot verify (secret rotation or tampering), or an - // unsigned row whose creator has exceeded the signing grace period. - const unsignedRequestIds = new Set(); - const invalidRequestIds: string[] = []; - for (const request of requests) { - const invocation = invocationById.get(request.invocationId); - if (!invocation) { - invalidRequestIds.push(request.id); - continue; - } - if (request.signedArguments === null) { - if ( - Date.now() - request.createdAt.getTime() >= - TOOL_ACTION_REQUEST_SIGNING_GRACE_MS - ) { - invalidRequestIds.push(request.id); - } else { - unsignedRequestIds.add(request.id); - } - continue; - } - let readable = false; - try { - readable = Boolean( - readSignedToolArgumentsPayload({ - signedArguments: request.signedArguments, - invocationId: invocation.id, - toolName: invocation.toolName, - }), - ); - } catch { - readable = false; - } - if (!readable) invalidRequestIds.push(request.id); - } - if (invalidRequestIds.length > 0) { - await db - .update(toolActionRequests) - .set({ - status: "cancelled", - resolvedAt: new Date(), - updatedAt: new Date(), - }) - .where( - and( - eq(toolActionRequests.companyId, companyId), - eq(toolActionRequests.status, "pending"), - inArray(toolActionRequests.id, invalidRequestIds), - ), - ); - } - const hiddenIds = new Set([ - ...invalidRequestIds, - ...unsignedRequestIds, - ]); - if (hiddenIds.size > 0) { - visibleRequests = requests.filter( - (request) => !hiddenIds.has(request.id), - ); - } - } - if (visibleRequests.length === 0) return []; - - const visibleInvocations = visibleRequests - .map((request) => invocationById.get(request.invocationId)) - .filter( - (invocation): invocation is typeof toolInvocations.$inferSelect => - Boolean(invocation), - ); - const connectionIds = [ - ...new Set( - visibleInvocations - .map((invocation) => invocation.connectionId) - .filter(Boolean), - ), - ] as string[]; - const connections = connectionIds.length - ? await db - .select() - .from(toolConnections) - .where(inArray(toolConnections.id, connectionIds)) - : []; - const connectionById = new Map( - connections.map((connection) => [connection.id, connection]), - ); - - const applicationIds = [ - ...new Set( - connections - .map((connection) => connection.applicationId) - .filter(Boolean), - ), - ] as string[]; - const applications = applicationIds.length - ? await db - .select() - .from(toolApplications) - .where(inArray(toolApplications.id, applicationIds)) - : []; - const applicationById = new Map( - applications.map((application) => [application.id, application]), - ); - - const catalogEntryIds = [ - ...new Set( - visibleInvocations - .map((invocation) => invocation.catalogEntryId) - .filter(Boolean), - ), - ] as string[]; - const catalogEntries = catalogEntryIds.length - ? await db - .select() - .from(toolCatalogEntries) - .where(inArray(toolCatalogEntries.id, catalogEntryIds)) - : []; - const catalogById = new Map( - catalogEntries.map((entry) => [entry.id, entry]), - ); - - return visibleRequests.map((request) => { - const invocation = invocationById.get(request.invocationId); - const connection = invocation?.connectionId - ? connectionById.get(invocation.connectionId) - : undefined; - const application = connection?.applicationId - ? applicationById.get(connection.applicationId) - : undefined; - const catalogEntry = invocation?.catalogEntryId - ? catalogById.get(invocation.catalogEntryId) - : undefined; - return { - request: toToolActionRequest(request), - toolName: invocation?.toolName ?? catalogEntry?.toolName ?? "", - toolTitle: catalogEntry?.title ?? null, - connectionId: connection?.id ?? invocation?.connectionId ?? null, - connectionName: connection?.name ?? null, - applicationName: application?.name ?? null, - riskLevel: catalogEntry?.riskLevel ?? null, - requestedByAgentId: request.requestedByAgentId ?? null, - }; - }); - }, + ...toolActionReviewQueue, ...profileService, ...profileResolver, mintConnectionTokenForAgent, - - listRuntimeSlots: async (companyId: string): Promise => { - const rows = await db - .select() - .from(toolRuntimeSlots) - .where(eq(toolRuntimeSlots.companyId, companyId)) - .orderBy(desc(toolRuntimeSlots.updatedAt)); - return rows.map(toRuntimeSlot); - }, - - stopRuntimeSlot: ( - companyId: string, - slotId: string, - actor?: ActorInfo, - ): Promise => - controlRuntimeSlot({ companyId, slotId, action: "stop", actor }), - - restartRuntimeSlot: ( - companyId: string, - slotId: string, - actor?: ActorInfo, - ): Promise => - controlRuntimeSlot({ companyId, slotId, action: "restart", actor }), + ...runtimeSlots.api, getRuntimeHealth: runtimeHealth, - getRunDecisionLookup: async ( - companyId: string, - runId: string, - ): Promise => { - const [run] = await db - .select({ id: heartbeatRuns.id }) - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.id, runId), - eq(heartbeatRuns.companyId, companyId), - ), - ) - .limit(1); - if (!run) throw notFound("Run not found"); - - const invocationRows = await db - .select() - .from(toolInvocations) - .where( - and( - eq(toolInvocations.companyId, companyId), - eq(toolInvocations.runId, runId), - ), - ) - .orderBy(desc(toolInvocations.createdAt)); - const invocationIds = invocationRows.map((row) => row.id); - const [actionRequestRows, auditEventRows] = - invocationIds.length > 0 - ? await Promise.all([ - db - .select() - .from(toolActionRequests) - .where( - and( - eq(toolActionRequests.companyId, companyId), - inArray(toolActionRequests.invocationId, invocationIds), - ), - ), - db - .select() - .from(toolCallEvents) - .where( - and( - eq(toolCallEvents.companyId, companyId), - eq(toolCallEvents.runId, runId), - inArray(toolCallEvents.invocationId, invocationIds), - ), - ) - .orderBy(desc(toolCallEvents.createdAt)), - ]) - : [[], []]; - - const actionRequestByInvocation = new Map( - actionRequestRows.map((row) => [row.invocationId, row]), - ); - const auditEventsByInvocation = new Map< - string, - (typeof toolCallEvents.$inferSelect)[] - >(); - for (const event of auditEventRows) { - if (!event.invocationId) continue; - const events = auditEventsByInvocation.get(event.invocationId) ?? []; - events.push(event); - auditEventsByInvocation.set(event.invocationId, events); - } - - const decisions: ToolRunDecision[] = invocationRows.map((invocation) => { - const actionRequest = - actionRequestByInvocation.get(invocation.id) ?? null; - const auditEvents = auditEventsByInvocation.get(invocation.id) ?? []; - const latestAuditEvent = auditEvents[0] ?? null; - const apiInvocation = toToolInvocation(invocation); - const apiActionRequest = actionRequest - ? toToolActionRequest(actionRequest) - : null; - const apiAuditEvents = auditEvents.map(toToolCallEvent); - const apiLatestAuditEvent = latestAuditEvent - ? toToolCallEvent(latestAuditEvent) - : null; - const pendingAction = - actionRequest && actionRequest.status === "pending" - ? { - actionRequestId: actionRequest.id, - issueId: actionRequest.issueId, - interactionId: actionRequest.interactionId, - approvalId: actionRequest.approvalId, - status: actionRequest.status, - previewMarkdown: actionRequest.previewMarkdown, - } - : null; - return { - invocation: apiInvocation, - actionRequest: apiActionRequest, - auditEvents: apiAuditEvents, - latestAuditEvent: apiLatestAuditEvent, - decision: latestAuditEvent?.decision ?? invocation.policyDecision, - outcome: latestAuditEvent?.outcome ?? null, - reasonCode: latestAuditEvent?.reasonCode ?? invocation.errorCode, - denialReason: denialReasonForDecision(invocation, latestAuditEvent), - pendingAction, - } satisfies ToolRunDecision; - }); - - return { runId, decisions }; - }, - previewMcpJsonImport: async ( input: ImportMcpJson, ): Promise => { diff --git a/server/src/services/tool-access/activity/projection.ts b/server/src/services/tool-access/activity/projection.ts new file mode 100644 index 00000000000..61d4a7ec2d2 --- /dev/null +++ b/server/src/services/tool-access/activity/projection.ts @@ -0,0 +1,138 @@ +import { toolActionRequests, toolCallEvents, toolInvocations } from "@paperclipai/db"; +import type { ToolActionRequest, ToolCallEvent, ToolInvocation } from "@paperclipai/shared"; + +export function toToolInvocation( + row: typeof toolInvocations.$inferSelect, +): ToolInvocation { + return { + id: row.id, + companyId: row.companyId, + idempotencyKey: row.idempotencyKey, + actorType: row.actorType as ToolInvocation["actorType"], + actorId: row.actorId, + agentId: row.agentId, + issueId: row.issueId, + runId: row.runId, + applicationId: row.applicationId, + connectionId: row.connectionId, + catalogEntryId: row.catalogEntryId, + toolName: row.toolName, + argumentsHash: row.argumentsHash, + argumentsSummary: row.argumentsSummary ?? null, + policyDecision: row.policyDecision, + matchedPolicyIds: row.matchedPolicyIds, + approvalState: row.approvalState, + status: row.status, + upstreamRequestId: row.upstreamRequestId, + resultHash: row.resultHash, + resultSummary: row.resultSummary ?? null, + resultSizeBytes: row.resultSizeBytes, + resultArtifactId: row.resultArtifactId, + errorCode: row.errorCode, + errorMessage: row.errorMessage, + startedAt: row.startedAt, + completedAt: row.completedAt, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} + +export function toToolActionRequest( + row: typeof toolActionRequests.$inferSelect, +): ToolActionRequest { + return { + id: row.id, + companyId: row.companyId, + invocationId: row.invocationId, + issueId: row.issueId, + interactionId: row.interactionId, + approvalId: row.approvalId, + status: row.status, + canonicalArgumentsHash: row.canonicalArgumentsHash, + canonicalArgumentsSummary: row.canonicalArgumentsSummary, + signedArguments: row.signedArguments, + previewMarkdown: row.previewMarkdown, + requestedByAgentId: row.requestedByAgentId, + requestedByUserId: row.requestedByUserId, + resolvedByAgentId: row.resolvedByAgentId, + resolvedByUserId: row.resolvedByUserId, + decidedByAgentId: row.decidedByAgentId, + decidedByUserId: row.decidedByUserId, + decidedAt: row.decidedAt, + expiresAt: row.expiresAt, + resolvedAt: row.resolvedAt, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} + +export function toToolCallEvent( + row: typeof toolCallEvents.$inferSelect, +): ToolCallEvent { + return { + id: row.id, + companyId: row.companyId, + eventType: row.eventType, + actorType: row.actorType as ToolCallEvent["actorType"], + actorId: row.actorId, + agentId: row.agentId, + runId: row.runId, + issueId: row.issueId, + applicationId: row.applicationId, + connectionId: row.connectionId, + catalogEntryId: row.catalogEntryId, + invocationId: row.invocationId, + actionRequestId: row.actionRequestId, + runtimeSlotId: row.runtimeSlotId, + toolName: row.toolName, + decision: row.decision, + matchedPolicyIds: row.matchedPolicyIds, + reasonCode: row.reasonCode, + outcome: row.outcome, + latencyMs: row.latencyMs, + argumentsSummary: row.argumentsSummary ?? null, + requestHash: row.requestHash, + requestSummary: row.requestSummary ?? null, + resultHash: row.resultHash, + resultSummary: row.resultSummary ?? null, + resultSizeBytes: row.resultSizeBytes, + redactionPlan: row.redactionPlan ?? null, + rateLimitState: row.rateLimitState ?? null, + metadata: row.metadata ?? null, + errorCode: row.errorCode, + errorMessage: row.errorMessage, + createdAt: row.createdAt, + }; +} + +export function userFallbackName(userId: string): string { + if (userId === "local-board") return "Board"; + return userId; +} + +export function denialReasonForDecision( + invocation: typeof toolInvocations.$inferSelect, + latestAuditEvent: typeof toolCallEvents.$inferSelect | null, +) { + if ( + invocation.status === "denied" || + invocation.status === "rate_limited" || + invocation.status === "failed" || + invocation.status === "timed_out" + ) { + return ( + invocation.errorMessage ?? + invocation.errorCode ?? + latestAuditEvent?.reasonCode ?? + null + ); + } + if ( + latestAuditEvent?.outcome === "denied" || + latestAuditEvent?.outcome === "failure" || + latestAuditEvent?.outcome === "timeout" + ) { + return latestAuditEvent.errorMessage ?? latestAuditEvent.reasonCode ?? null; + } + return null; +} diff --git a/server/src/services/tool-access/activity/review-queue.ts b/server/src/services/tool-access/activity/review-queue.ts new file mode 100644 index 00000000000..2eb5d1e4c9e --- /dev/null +++ b/server/src/services/tool-access/activity/review-queue.ts @@ -0,0 +1,197 @@ +import { and, desc, eq, inArray } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { toolApplications, toolActionRequests, toolCatalogEntries, toolConnections, toolInvocations } from "@paperclipai/db"; +import type { ToolActionRequestListItem, ToolActionRequestStatus } from "@paperclipai/shared"; +import { readSignedToolArgumentsPayload, TOOL_ACTION_REQUEST_SIGNING_GRACE_MS } from "../../tool-content-guards.js"; +import { toToolActionRequest } from "./projection.js"; +/** Pending requests stay hidden until signing completes; invalid signatures are cancelled. */ +export function createToolActionReviewQueue(db: Db) { + return { + /** + * List "Ask first" action requests for the review queue, enriched with the + * connection/app context the prosumer card renders. Defaults to pending. + */ +listActionRequests: async ( + companyId: string, + status: ToolActionRequestStatus = "pending", + ): Promise => { + const requests = await db + .select() + .from(toolActionRequests) + .where( + and( + eq(toolActionRequests.companyId, companyId), + eq(toolActionRequests.status, status), + ), + ) + .orderBy(desc(toolActionRequests.createdAt)); + if (requests.length === 0) return []; + + const invocationIds = [ + ...new Set(requests.map((request) => request.invocationId)), + ]; + const invocations = await db + .select() + .from(toolInvocations) + .where( + and( + eq(toolInvocations.companyId, companyId), + inArray(toolInvocations.id, invocationIds), + ), + ); + const invocationById = new Map( + invocations.map((invocation) => [invocation.id, invocation]), + ); + let visibleRequests = requests; + if (status === "pending") { + // A pending request that the creator has not signed yet is still being + // set up. The gateway creates the row (signedArguments = null) and signs + // it in a second step, so a review-queue read can observe the row inside + // that window. Hide such a request from the queue, but do not cancel it — + // cancelling here races the two-step create and makes the later approve + // fail with action_not_pending. Only cancel a request that carries a + // signature we cannot verify (secret rotation or tampering), or an + // unsigned row whose creator has exceeded the signing grace period. + const unsignedRequestIds = new Set(); + const invalidRequestIds: string[] = []; + for (const request of requests) { + const invocation = invocationById.get(request.invocationId); + if (!invocation) { + invalidRequestIds.push(request.id); + continue; + } + if (request.signedArguments === null) { + if ( + Date.now() - request.createdAt.getTime() >= + TOOL_ACTION_REQUEST_SIGNING_GRACE_MS + ) { + invalidRequestIds.push(request.id); + } else { + unsignedRequestIds.add(request.id); + } + continue; + } + let readable = false; + try { + readable = Boolean( + readSignedToolArgumentsPayload({ + signedArguments: request.signedArguments, + invocationId: invocation.id, + toolName: invocation.toolName, + }), + ); + } catch { + readable = false; + } + if (!readable) invalidRequestIds.push(request.id); + } + if (invalidRequestIds.length > 0) { + await db + .update(toolActionRequests) + .set({ + status: "cancelled", + resolvedAt: new Date(), + updatedAt: new Date(), + }) + .where( + and( + eq(toolActionRequests.companyId, companyId), + eq(toolActionRequests.status, "pending"), + inArray(toolActionRequests.id, invalidRequestIds), + ), + ); + } + const hiddenIds = new Set([ + ...invalidRequestIds, + ...unsignedRequestIds, + ]); + if (hiddenIds.size > 0) { + visibleRequests = requests.filter( + (request) => !hiddenIds.has(request.id), + ); + } + } + if (visibleRequests.length === 0) return []; + + const visibleInvocations = visibleRequests + .map((request) => invocationById.get(request.invocationId)) + .filter( + (invocation): invocation is typeof toolInvocations.$inferSelect => + Boolean(invocation), + ); + const connectionIds = [ + ...new Set( + visibleInvocations + .map((invocation) => invocation.connectionId) + .filter(Boolean), + ), + ] as string[]; + const connections = connectionIds.length + ? await db + .select() + .from(toolConnections) + .where(inArray(toolConnections.id, connectionIds)) + : []; + const connectionById = new Map( + connections.map((connection) => [connection.id, connection]), + ); + + const applicationIds = [ + ...new Set( + connections + .map((connection) => connection.applicationId) + .filter(Boolean), + ), + ] as string[]; + const applications = applicationIds.length + ? await db + .select() + .from(toolApplications) + .where(inArray(toolApplications.id, applicationIds)) + : []; + const applicationById = new Map( + applications.map((application) => [application.id, application]), + ); + + const catalogEntryIds = [ + ...new Set( + visibleInvocations + .map((invocation) => invocation.catalogEntryId) + .filter(Boolean), + ), + ] as string[]; + const catalogEntries = catalogEntryIds.length + ? await db + .select() + .from(toolCatalogEntries) + .where(inArray(toolCatalogEntries.id, catalogEntryIds)) + : []; + const catalogById = new Map( + catalogEntries.map((entry) => [entry.id, entry]), + ); + + return visibleRequests.map((request) => { + const invocation = invocationById.get(request.invocationId); + const connection = invocation?.connectionId + ? connectionById.get(invocation.connectionId) + : undefined; + const application = connection?.applicationId + ? applicationById.get(connection.applicationId) + : undefined; + const catalogEntry = invocation?.catalogEntryId + ? catalogById.get(invocation.catalogEntryId) + : undefined; + return { + request: toToolActionRequest(request), + toolName: invocation?.toolName ?? catalogEntry?.toolName ?? "", + toolTitle: catalogEntry?.title ?? null, + connectionId: connection?.id ?? invocation?.connectionId ?? null, + connectionName: connection?.name ?? null, + applicationName: application?.name ?? null, + riskLevel: catalogEntry?.riskLevel ?? null, + requestedByAgentId: request.requestedByAgentId ?? null, + }; + }); + } + }; +} diff --git a/server/src/services/tool-access/activity/service.ts b/server/src/services/tool-access/activity/service.ts new file mode 100644 index 00000000000..49ff10acdbb --- /dev/null +++ b/server/src/services/tool-access/activity/service.ts @@ -0,0 +1,339 @@ +import { and, desc, eq, gte, inArray } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { agents, connectionTokenIssuances, authUsers, heartbeatRuns, issues, toolActionRequests, toolConnections, toolCallEvents, toolInvocations } from "@paperclipai/db"; +import type { ToolConnectionActivityResponse, ToolRunDecision, ToolRunDecisionLookup } from "@paperclipai/shared"; +import { notFound } from "../../../errors.js"; +import { listConnectionLifecycleEvents } from "../../tool-connection-activity.js"; +import { toToolInvocation, toToolActionRequest, toToolCallEvent, userFallbackName, denialReasonForDecision } from "./projection.js"; +type ConnectionLookup = (idOrUid: string, companyId?: string) => Promise; + +export function createToolActivityService(db: Db, getConnectionRow: ConnectionLookup) { + return { + getConnectionUsage: async ( + idOrUid: string, + range: "7d" | "30d", + companyId?: string, + ) => { + const connection = await getConnectionRow(idOrUid, companyId); + const days = range === "30d" ? 30 : 7; + const start = new Date(); + start.setUTCHours(0, 0, 0, 0); + start.setUTCDate(start.getUTCDate() - days + 1); + const [issuances, invocations] = await Promise.all([ + db + .select({ + createdAt: connectionTokenIssuances.createdAt, + outcome: connectionTokenIssuances.outcome, + path: connectionTokenIssuances.path, + }) + .from(connectionTokenIssuances) + .where( + and( + eq(connectionTokenIssuances.companyId, connection.companyId), + eq(connectionTokenIssuances.connectionId, connection.id), + gte(connectionTokenIssuances.createdAt, start), + ), + ), + db + .select({ + createdAt: toolInvocations.createdAt, + riskLevel: toolInvocations.riskLevel, + }) + .from(toolInvocations) + .where( + and( + eq(toolInvocations.companyId, connection.companyId), + eq(toolInvocations.connectionId, connection.id), + gte(toolInvocations.createdAt, start), + ), + ), + ]); + const buckets = Array.from({ length: days }, (_, offset) => { + const date = new Date(start); + date.setUTCDate(start.getUTCDate() + offset); + return { + date: date.toISOString().slice(0, 10), + issuances: { + total: 0, + byOutcome: {} as Record, + byPath: {} as Record, + }, + invocations: { total: 0, byRiskLevel: {} as Record }, + deliveries: { received: 0, forwarded: 0 }, + }; + }); + const byDate = new Map(buckets.map((bucket) => [bucket.date, bucket])); + for (const row of issuances) { + const bucket = byDate.get(row.createdAt.toISOString().slice(0, 10)); + if (!bucket) continue; + bucket.issuances.total += 1; + bucket.issuances.byOutcome[row.outcome] = + (bucket.issuances.byOutcome[row.outcome] ?? 0) + 1; + bucket.issuances.byPath[row.path] = + (bucket.issuances.byPath[row.path] ?? 0) + 1; + } + for (const row of invocations) { + const bucket = byDate.get(row.createdAt.toISOString().slice(0, 10)); + if (!bucket) continue; + const riskLevel = row.riskLevel ?? "unknown"; + bucket.invocations.total += 1; + bucket.invocations.byRiskLevel[riskLevel] = + (bucket.invocations.byRiskLevel[riskLevel] ?? 0) + 1; + } + return { + connection: { id: connection.id, uid: connection.uid }, + range, + buckets, + }; + }, + + /** Recent tool-call events for one connection — drives App detail · Recent activity. */ +listConnectionActivity: async ( + connectionId: string, + companyId?: string, + limit = 20, + ): Promise => { + const connection = await getConnectionRow(connectionId, companyId); + const safeLimit = Math.max(1, Math.min(100, Math.floor(limit))); + const rows = await db + .select() + .from(toolCallEvents) + .where( + and( + eq(toolCallEvents.companyId, connection.companyId), + eq(toolCallEvents.connectionId, connection.id), + ), + ) + .orderBy(desc(toolCallEvents.createdAt)) + .limit(safeLimit); + const events = rows.map(toToolCallEvent); + + const issueIds = [ + ...new Set(rows.map((row) => row.issueId).filter(Boolean)), + ] as string[]; + const issueRows = issueIds.length + ? await db + .select({ + id: issues.id, + identifier: issues.identifier, + title: issues.title, + }) + .from(issues) + .where( + and( + eq(issues.companyId, connection.companyId), + inArray(issues.id, issueIds), + ), + ) + : []; + const issueMap = Object.fromEntries( + issueRows.map((issue) => [ + issue.id, + { + identifier: issue.identifier ?? issue.id, + title: issue.title, + }, + ]), + ); + + const actionRequestIds = [ + ...new Set(rows.map((row) => row.actionRequestId).filter(Boolean)), + ] as string[]; + const requestRows = actionRequestIds.length + ? await db + .select({ + id: toolActionRequests.id, + status: toolActionRequests.status, + resolvedByAgentId: toolActionRequests.resolvedByAgentId, + resolvedByUserId: toolActionRequests.resolvedByUserId, + }) + .from(toolActionRequests) + .where( + and( + eq(toolActionRequests.companyId, connection.companyId), + inArray(toolActionRequests.id, actionRequestIds), + ), + ) + : []; + + const resolverAgentIds = [ + ...new Set( + requestRows.map((row) => row.resolvedByAgentId).filter(Boolean), + ), + ] as string[]; + const resolverUserIds = [ + ...new Set( + requestRows.map((row) => row.resolvedByUserId).filter(Boolean), + ), + ] as string[]; + const resolverAgents = resolverAgentIds.length + ? await db + .select({ id: agents.id, name: agents.name }) + .from(agents) + .where( + and( + eq(agents.companyId, connection.companyId), + inArray(agents.id, resolverAgentIds), + ), + ) + : []; + const resolverUsers = resolverUserIds.length + ? await db + .select({ + id: authUsers.id, + name: authUsers.name, + email: authUsers.email, + }) + .from(authUsers) + .where(inArray(authUsers.id, resolverUserIds)) + : []; + const resolverAgentNames = new Map( + resolverAgents.map((agent) => [agent.id, agent.name]), + ); + const resolverUserNames = new Map( + resolverUsers.map((user) => [ + user.id, + user.name?.trim() || user.email?.trim() || user.id, + ]), + ); + const actionRequestMap = Object.fromEntries( + requestRows.map((request) => [ + request.id, + { + status: request.status, + resolverDisplayName: request.resolvedByAgentId + ? (resolverAgentNames.get(request.resolvedByAgentId) ?? + request.resolvedByAgentId) + : request.resolvedByUserId + ? (resolverUserNames.get(request.resolvedByUserId) ?? + userFallbackName(request.resolvedByUserId)) + : null, + resolvedByAgentId: request.resolvedByAgentId, + resolvedByUserId: request.resolvedByUserId, + }, + ]), + ); + + const lifecycleEvents = await listConnectionLifecycleEvents(db, { + companyId: connection.companyId, + connectionIds: [connection.id], + limit: safeLimit, + }); + + return { + connectionId: connection.id, + events, + lifecycleEvents, + issues: issueMap, + actionRequests: actionRequestMap, + }; + }, + + getRunDecisionLookup: async ( + companyId: string, + runId: string, + ): Promise => { + const [run] = await db + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, runId), + eq(heartbeatRuns.companyId, companyId), + ), + ) + .limit(1); + if (!run) throw notFound("Run not found"); + + const invocationRows = await db + .select() + .from(toolInvocations) + .where( + and( + eq(toolInvocations.companyId, companyId), + eq(toolInvocations.runId, runId), + ), + ) + .orderBy(desc(toolInvocations.createdAt)); + const invocationIds = invocationRows.map((row) => row.id); + const [actionRequestRows, auditEventRows] = + invocationIds.length > 0 + ? await Promise.all([ + db + .select() + .from(toolActionRequests) + .where( + and( + eq(toolActionRequests.companyId, companyId), + inArray(toolActionRequests.invocationId, invocationIds), + ), + ), + db + .select() + .from(toolCallEvents) + .where( + and( + eq(toolCallEvents.companyId, companyId), + eq(toolCallEvents.runId, runId), + inArray(toolCallEvents.invocationId, invocationIds), + ), + ) + .orderBy(desc(toolCallEvents.createdAt)), + ]) + : [[], []]; + + const actionRequestByInvocation = new Map( + actionRequestRows.map((row) => [row.invocationId, row]), + ); + const auditEventsByInvocation = new Map< + string, + (typeof toolCallEvents.$inferSelect)[] + >(); + for (const event of auditEventRows) { + if (!event.invocationId) continue; + const events = auditEventsByInvocation.get(event.invocationId) ?? []; + events.push(event); + auditEventsByInvocation.set(event.invocationId, events); + } + + const decisions: ToolRunDecision[] = invocationRows.map((invocation) => { + const actionRequest = + actionRequestByInvocation.get(invocation.id) ?? null; + const auditEvents = auditEventsByInvocation.get(invocation.id) ?? []; + const latestAuditEvent = auditEvents[0] ?? null; + const apiInvocation = toToolInvocation(invocation); + const apiActionRequest = actionRequest + ? toToolActionRequest(actionRequest) + : null; + const apiAuditEvents = auditEvents.map(toToolCallEvent); + const apiLatestAuditEvent = latestAuditEvent + ? toToolCallEvent(latestAuditEvent) + : null; + const pendingAction = + actionRequest && actionRequest.status === "pending" + ? { + actionRequestId: actionRequest.id, + issueId: actionRequest.issueId, + interactionId: actionRequest.interactionId, + approvalId: actionRequest.approvalId, + status: actionRequest.status, + previewMarkdown: actionRequest.previewMarkdown, + } + : null; + return { + invocation: apiInvocation, + actionRequest: apiActionRequest, + auditEvents: apiAuditEvents, + latestAuditEvent: apiLatestAuditEvent, + decision: latestAuditEvent?.decision ?? invocation.policyDecision, + outcome: latestAuditEvent?.outcome ?? null, + reasonCode: latestAuditEvent?.reasonCode ?? invocation.errorCode, + denialReason: denialReasonForDecision(invocation, latestAuditEvent), + pendingAction, + } satisfies ToolRunDecision; + }); + + return { runId, decisions }; + } + }; +} diff --git a/server/src/services/tool-access/runtime/health.ts b/server/src/services/tool-access/runtime/health.ts new file mode 100644 index 00000000000..40f8256cb59 --- /dev/null +++ b/server/src/services/tool-access/runtime/health.ts @@ -0,0 +1,426 @@ +import { and, desc, eq, gte, sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { toolAccessAuditEvents, toolConnections, toolCallEvents, toolRuntimeMetricCounters, toolRuntimeSlots } from "@paperclipai/db"; +import type { DeploymentExposure, DeploymentMode, ToolRuntimeAlertRecommendation, ToolRuntimeHealthSummary } from "@paperclipai/shared"; +import { TOOL_RUNTIME_AUDIT_WRITE_FAILURE_METRIC } from "../../tool-runtime-metrics.js"; + +function asRecord(value: unknown): Record { + if (value && typeof value === "object" && !Array.isArray(value)) + return value as Record; + return {}; +} + +function runtimeAlert( + input: ToolRuntimeAlertRecommendation, +): ToolRuntimeAlertRecommendation { + return input; +} + +function buildRuntimeAlerts(input: { + stuckStartingSlots: number; + stuckRunningSlots: number; + timeoutRate: number; + timeoutCount: number; + failureRate: number; + failureCount: number; + capacityDeferrals: number; + restartAttempts: number; + restartSuppressions: number; + degradedConnections: number; + disabledConnections: number; + missingSecretFailures: number; + auditWriteFailures: number; +}): ToolRuntimeAlertRecommendation[] { + const runbookSection = "doc/MCP-RUNTIME-OPERATIONS.md"; + const timeoutSeverity = + input.timeoutCount >= 10 || input.timeoutRate >= 25 + ? "critical" + : input.timeoutCount >= 3 && input.timeoutRate >= 10 + ? "warning" + : "warning"; + const failureSeverity = + input.failureCount >= 10 || input.failureRate >= 25 + ? "critical" + : input.failureCount >= 5 && input.failureRate >= 10 + ? "warning" + : "warning"; + const restartSeverity = + input.restartSuppressions > 0 ? "critical" : "warning"; + return [ + runtimeAlert({ + name: "mcp_runtime_stuck_starting_slot", + severity: "critical", + status: input.stuckStartingSlots > 0 ? "firing" : "ok", + threshold: "Any starting slot older than 5 minutes.", + observed: `${input.stuckStartingSlots} stuck starting slot(s).`, + description: + "A local stdio runtime slot is stuck before it reaches running state.", + firstResponderAction: + "Inspect the slot health/logs, stop the slot, restart it once, then disable the connection if the slot sticks again.", + runbookSection, + }), + runtimeAlert({ + name: "mcp_runtime_stuck_running_slot", + severity: "critical", + status: input.stuckRunningSlots > 0 ? "firing" : "ok", + threshold: "Any running slot with no progress for 5 minutes.", + observed: `${input.stuckRunningSlots} stuck running slot(s).`, + description: + "A runtime slot is running but has not recorded progress inside the supervisor stuck-slot window.", + firstResponderAction: + "Inspect recent audit events and active tool calls; restart the slot only after confirming no healthy call is still in progress.", + runbookSection, + }), + runtimeAlert({ + name: "mcp_runtime_high_timeout_rate", + severity: timeoutSeverity, + status: + input.timeoutCount >= 3 && input.timeoutRate >= 10 ? "firing" : "ok", + threshold: + "Warning at >=3 timeouts and >=10% timeout rate in 1 hour; critical at >=10 timeouts or >=25%.", + observed: `${input.timeoutCount} timeout(s), ${input.timeoutRate}% timeout rate.`, + description: + "Tool gateway calls are timing out or being runtime-deferred at an elevated rate.", + firstResponderAction: + "Check upstream MCP health, Paperclip runtime capacity, and recent gateway audit failures before retrying workloads.", + runbookSection, + }), + runtimeAlert({ + name: "mcp_runtime_high_error_rate", + severity: failureSeverity, + status: + input.failureCount >= 5 && input.failureRate >= 10 ? "firing" : "ok", + threshold: + "Warning at >=5 failures and >=10% failure rate in 1 hour; critical at >=10 failures or >=25%.", + observed: `${input.failureCount} failure(s), ${input.failureRate}% failure rate.`, + description: + "Tool gateway calls are failing after policy authorization.", + firstResponderAction: + "Group audit failures by reasonCode, then fix credentials/config or disable the affected connection.", + runbookSection, + }), + runtimeAlert({ + name: "mcp_runtime_capacity_deferrals_repeated", + severity: input.capacityDeferrals >= 10 ? "critical" : "warning", + status: input.capacityDeferrals >= 3 ? "firing" : "ok", + threshold: + "Warning at >=3 capacity deferrals in 1 hour; critical at >=10.", + observed: `${input.capacityDeferrals} capacity deferral(s) in 1 hour.`, + description: + "The runtime supervisor is refusing local stdio work because company or host slot capacity is exhausted.", + firstResponderAction: + "Stop idle/stale slots, lower noisy workloads, or raise slot caps only after confirming host capacity.", + runbookSection, + }), + runtimeAlert({ + name: "mcp_runtime_restart_storm", + severity: restartSeverity, + status: + input.restartSuppressions > 0 || input.restartAttempts >= 3 + ? "firing" + : "ok", + threshold: + "Warning at >=3 restarts in 1 hour; critical on any restart suppression.", + observed: `${input.restartAttempts} restart attempt(s), ${input.restartSuppressions} suppression(s).`, + description: + "Runtime slots are restarting repeatedly or have hit restart-storm suppression.", + firstResponderAction: + "Stop the affected slot, inspect stderr/audit reason codes, and keep the connection disabled until the template/upstream is fixed.", + runbookSection, + }), + runtimeAlert({ + name: "mcp_runtime_connection_health_degraded", + severity: input.degradedConnections > 0 ? "critical" : "warning", + status: + input.degradedConnections > 0 || input.disabledConnections > 0 + ? "firing" + : "ok", + threshold: + "Any active enabled connection with degraded/failed/missing-secret health, or any disabled enabled-path connection.", + observed: `${input.degradedConnections} degraded connection(s), ${input.disabledConnections} disabled connection(s).`, + description: + "A configured MCP connection is not healthy or has been disabled.", + firstResponderAction: + "Run a connection health check, refresh catalog after recovery, or keep the connection disabled and route agents to alternatives.", + runbookSection, + }), + runtimeAlert({ + name: "mcp_runtime_missing_secret_failures", + severity: input.missingSecretFailures >= 3 ? "critical" : "warning", + status: input.missingSecretFailures > 0 ? "firing" : "ok", + threshold: + "Warning on any missing-secret failure; critical at >=3 in 1 hour.", + observed: `${input.missingSecretFailures} missing-secret failure(s) in 1 hour.`, + description: + "A connection or tool call needed a bound secret that could not be resolved.", + firstResponderAction: + "Check secret bindings and provider health without printing secret values; rotate or rebind missing secrets.", + runbookSection, + }), + runtimeAlert({ + name: "mcp_runtime_audit_write_failures", + severity: "critical", + status: input.auditWriteFailures > 0 ? "firing" : "ok", + threshold: "Any audit write failure.", + observed: `${input.auditWriteFailures} audit write failure(s) in 1 hour.`, + description: + "Tool gateway audit writes failed, reducing incident traceability.", + firstResponderAction: + "Treat as a control-plane incident: check database writes, activity log writes, and retry only after audit durability is restored.", + runbookSection, + }), + ]; +} + +function numberValue(value: unknown): number | null { + const parsed = typeof value === "number" ? value : Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function percent(numerator: number, denominator: number): number { + if (denominator <= 0) return 0; + return Math.round((numerator / denominator) * 1000) / 10; +} + +function percentile(values: number[], p: number): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min( + sorted.length - 1, + Math.max(0, Math.ceil((p / 100) * sorted.length) - 1), + ); + return sorted[index] ?? null; +} + +type RuntimeHealthOptions = { deploymentMode?: DeploymentMode; deploymentExposure?: DeploymentExposure }; + +export function createToolRuntimeHealth(db: Db, options: RuntimeHealthOptions, now: () => Date, trustedRuntimeHost: () => string | null) { + async function runtimeHealth( + companyId: string, + ): Promise { + const generatedAt = now(); + const windowStartedAt = new Date(generatedAt.getTime() - 60 * 60 * 1000); + const stuckSlotMs = 5 * 60 * 1000; + const [ + slots, + connections, + auditRows, + callEvents, + auditWriteFailureCounterRows, + ] = await Promise.all([ + db + .select() + .from(toolRuntimeSlots) + .where(eq(toolRuntimeSlots.companyId, companyId)), + db + .select() + .from(toolConnections) + .where(eq(toolConnections.companyId, companyId)), + db + .select() + .from(toolAccessAuditEvents) + .where( + and( + eq(toolAccessAuditEvents.companyId, companyId), + gte(toolAccessAuditEvents.createdAt, windowStartedAt), + ), + ) + .orderBy(desc(toolAccessAuditEvents.createdAt)), + db + .select() + .from(toolCallEvents) + .where( + and( + eq(toolCallEvents.companyId, companyId), + gte(toolCallEvents.createdAt, windowStartedAt), + ), + ) + .orderBy(desc(toolCallEvents.createdAt)), + db + .select({ + count: sql`coalesce(sum(${toolRuntimeMetricCounters.count}), 0)::int`, + }) + .from(toolRuntimeMetricCounters) + .where( + and( + eq(toolRuntimeMetricCounters.companyId, companyId), + eq( + toolRuntimeMetricCounters.metric, + TOOL_RUNTIME_AUDIT_WRITE_FAILURE_METRIC, + ), + gte(toolRuntimeMetricCounters.bucketStartAt, windowStartedAt), + ), + ), + ]); + const activeSlots = slots.filter( + (slot) => + slot.status === "starting" || + slot.status === "running" || + slot.status === "idle", + ); + const staleActiveSlots = activeSlots.filter((slot) => { + const lastProgressAt = + slot.lastUsedAt ?? slot.startedAt ?? slot.updatedAt; + return generatedAt.getTime() - lastProgressAt.getTime() > stuckSlotMs; + }); + const callTerminalEvents = callEvents.filter( + (event) => + event.eventType === "call_completed" || + event.eventType === "call_failed" || + event.eventType === "call_denied", + ); + const toolCallsLastHour = callTerminalEvents.length; + const toolTimeoutsLastHour = callTerminalEvents.filter( + (event) => event.outcome === "timeout", + ).length; + const toolFailuresLastHour = callTerminalEvents.filter( + (event) => event.outcome === "failure", + ).length; + const durations = auditRows + .map((row) => numberValue(asRecord(row.details).durationMs)) + .filter((value): value is number => value !== null && value >= 0); + const capacityDeferrals = auditRows.filter( + (row) => + row.action === "runtime_deferred" || + row.reasonCode === "runtime_company_capacity_exhausted" || + row.reasonCode === "runtime_host_capacity_exhausted", + ).length; + const restartAttempts = auditRows.filter( + (row) => + row.action === "runtime_started" && row.reasonCode !== "lazy_start", + ).length; + const restartSuppressions = auditRows.filter( + (row) => + row.action === "runtime_restart_suppressed" || + row.reasonCode === "runtime_restart_suppressed", + ).length; + const idleEvictions = auditRows.filter( + (row) => + row.action === "runtime_stopped" && + row.reasonCode === "idle_ttl_expired", + ).length; + const missingSecretFailures = auditRows.filter( + (row) => + row.reasonCode === "missing_secret" || + (row.outcome === "failure" && row.reasonCode?.includes("secret")), + ).length; + const legacyAuditWriteFailures = auditRows.filter( + (row) => + row.action === "runtime_audit_write_failed" || + row.reasonCode === "audit_write_failed", + ).length; + const auditWriteFailuresMetric = + Number(auditWriteFailureCounterRows[0]?.count ?? 0) + + legacyAuditWriteFailures; + const enabledPathConnections = connections.filter( + (connection) => connection.status === "active" && connection.enabled, + ); + const activeConnections = enabledPathConnections.length; + const disabledConnections = connections.filter( + (connection) => connection.status === "disabled", + ).length; + const degradedConnections = enabledPathConnections.filter((connection) => + ["degraded", "failed", "error", "missing_secret"].includes( + connection.healthStatus, + ), + ).length; + const metrics = { + windowStartedAt, + windowEndedAt: generatedAt, + activeSlots: activeSlots.length, + startingSlots: slots.filter((slot) => slot.status === "starting").length, + runningSlots: slots.filter((slot) => slot.status === "running").length, + idleSlots: slots.filter((slot) => slot.status === "idle").length, + failedSlots: slots.filter( + (slot) => slot.status === "failed" || slot.status === "error", + ).length, + stoppedSlots: slots.filter( + (slot) => slot.status === "stopped" || slot.status === "disabled", + ).length, + stuckStartingSlots: staleActiveSlots.filter( + (slot) => slot.status === "starting", + ).length, + stuckRunningSlots: staleActiveSlots.filter( + (slot) => slot.status === "running", + ).length, + capacityDeferralsLastHour: capacityDeferrals, + restartAttemptsLastHour: restartAttempts, + restartSuppressionsLastHour: restartSuppressions, + idleEvictionsLastHour: idleEvictions, + toolCallsLastHour, + toolTimeoutsLastHour, + toolFailuresLastHour, + timeoutRateLastHour: percent(toolTimeoutsLastHour, toolCallsLastHour), + failureRateLastHour: percent(toolFailuresLastHour, toolCallsLastHour), + averageToolLatencyMsLastHour: + durations.length > 0 + ? Math.round( + durations.reduce((sum, value) => sum + value, 0) / + durations.length, + ) + : null, + p95ToolLatencyMsLastHour: percentile(durations, 95), + missingSecretFailuresLastHour: missingSecretFailures, + auditWriteFailuresLastHour: auditWriteFailuresMetric, + activeConnections, + disabledConnections, + degradedConnections, + remoteHttpConnections: connections.filter( + (connection) => + connection.status !== "archived" && + connection.transport === "mcp_remote", + ).length, + localStdioConnections: connections.filter( + (connection) => + connection.status !== "archived" && + connection.transport === "local_stdio", + ).length, + }; + const recommendations = buildRuntimeAlerts({ + stuckStartingSlots: metrics.stuckStartingSlots, + stuckRunningSlots: metrics.stuckRunningSlots, + timeoutRate: metrics.timeoutRateLastHour, + timeoutCount: metrics.toolTimeoutsLastHour, + failureRate: metrics.failureRateLastHour, + failureCount: metrics.toolFailuresLastHour, + capacityDeferrals, + restartAttempts, + restartSuppressions, + degradedConnections, + disabledConnections, + missingSecretFailures, + auditWriteFailures: metrics.auditWriteFailuresLastHour, + }); + const firing = recommendations.filter((alert) => alert.status === "firing"); + const status = firing.some((alert) => alert.severity === "critical") + ? "critical" + : firing.length > 0 + ? "degraded" + : "ok"; + const deploymentMode = options.deploymentMode ?? "local_trusted"; + const deploymentExposure = options.deploymentExposure ?? "private"; + const localStdioSupported = + deploymentMode === "local_trusted" || Boolean(trustedRuntimeHost()); + return { + status, + generatedAt, + runbookPath: "doc/MCP-RUNTIME-OPERATIONS.md", + metrics, + supportMatrix: { + remoteHttp: { + supported: true, + note: "mcp_remote MCP connections are supported in hosted cloud and local deployments.", + }, + localStdio: { + supported: localStdioSupported, + note: localStdioSupported + ? "local_stdio is available for local trusted mode or through the configured trusted MCP runtime host." + : `local_stdio should stay disabled for ${deploymentMode}/${deploymentExposure}; use mcp_remote or configure a trusted runtime worker.`, + }, + }, + alerts: firing, + recommendations, + }; + } + return runtimeHealth; +} diff --git a/server/src/services/tool-access/runtime/slots.ts b/server/src/services/tool-access/runtime/slots.ts new file mode 100644 index 00000000000..3ca8e6b4d81 --- /dev/null +++ b/server/src/services/tool-access/runtime/slots.ts @@ -0,0 +1,182 @@ +import type { ActorInfo } from "../types.js"; +import { and, desc, eq } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { toolConnections, toolRuntimeSlots } from "@paperclipai/db"; +import type { ToolRuntimeSlot } from "@paperclipai/shared"; +import { HttpError, notFound } from "../../../errors.js"; +import { logActivity } from "../../activity-log.js"; +import { createToolRuntimeSupervisor, ToolRuntimeSupervisorError } from "../../tool-runtime-supervisor.js"; + +function toRuntimeSlot( + row: typeof toolRuntimeSlots.$inferSelect, +): ToolRuntimeSlot { + return { + id: row.id, + companyId: row.companyId, + applicationId: row.applicationId, + connectionId: row.connectionId, + projectWorkspaceId: row.projectWorkspaceId, + executionWorkspaceId: row.executionWorkspaceId, + issueId: row.issueId, + ownerScopeType: row.ownerScopeType, + ownerScopeId: row.ownerScopeId, + runtimeKind: row.runtimeKind, + slotKey: row.slotKey, + status: row.status, + reuseKey: row.reuseKey, + workspaceScope: row.workspaceScope, + credentialScopeHash: row.credentialScopeHash, + provider: row.provider, + providerRef: row.providerRef, + processId: row.processId, + commandTemplateKey: row.commandTemplateKey, + healthStatus: row.healthStatus, + healthMessage: row.healthMessage, + lastHealthCheckAt: row.lastHealthCheckAt, + lastStartedAt: row.lastStartedAt, + startedAt: row.startedAt, + stoppedAt: row.stoppedAt, + lastUsedAt: row.lastUsedAt, + idleExpiresAt: row.idleExpiresAt, + idleDeadlineAt: row.idleDeadlineAt, + lastError: row.lastError, + metadata: row.metadata ?? null, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} + +export function createToolRuntimeSlots(db: Db, runtimeSupervisor: Pick, "stopSlot" | "restartSlot">) { + async function ensureRuntimeSlot( + connection: typeof toolConnections.$inferSelect, + ): Promise { + if (connection.transport !== "local_stdio" || connection.config.templateId === "paperclip.cognee-cloud") return null; + const slotKey = `mcp:${connection.companyId}:${connection.id}`; + const [existing] = await db + .select() + .from(toolRuntimeSlots) + .where( + and( + eq(toolRuntimeSlots.companyId, connection.companyId), + eq(toolRuntimeSlots.slotKey, slotKey), + ), + ); + if (existing) return toRuntimeSlot(existing); + const [created] = await db + .insert(toolRuntimeSlots) + .values({ + companyId: connection.companyId, + applicationId: connection.applicationId, + connectionId: connection.id, + slotKey, + ownerScopeType: "connection", + ownerScopeId: connection.id, + runtimeKind: "local_stdio", + status: "stopped", + provider: "paperclip", + providerRef: `template:${String(connection.config.templateId)}`, + commandTemplateKey: String(connection.config.templateId), + healthStatus: "unchecked", + metadata: { templateId: connection.config.templateId }, + }) + .returning(); + return toRuntimeSlot(created); + } + + async function runtimeSlotById( + companyId: string, + slotId: string, + ): Promise { + const [row] = await db + .select() + .from(toolRuntimeSlots) + .where( + and( + eq(toolRuntimeSlots.companyId, companyId), + eq(toolRuntimeSlots.id, slotId), + ), + ) + .limit(1); + if (!row) throw notFound("Runtime slot not found"); + return toRuntimeSlot(row); + } + + function runtimeSupervisorHttpError(error: ToolRuntimeSupervisorError) { + return new HttpError(error.status, error.message, { + code: error.reasonCode, + ...error.details, + }); + } + + async function controlRuntimeSlot(input: { + companyId: string; + slotId: string; + action: "stop" | "restart"; + actor?: ActorInfo; + }): Promise { + try { + if (input.action === "stop") { + await runtimeSupervisor.stopSlot({ + companyId: input.companyId, + slotId: input.slotId, + reason: "operator_stop", + }); + } else { + await runtimeSupervisor.restartSlot({ + companyId: input.companyId, + slotId: input.slotId, + }); + } + const slot = await runtimeSlotById(input.companyId, input.slotId); + await logActivity(db, { + companyId: input.companyId, + actorType: input.actor?.actorType ?? "system", + actorId: input.actor?.actorId ?? "tool-access-service", + action: + input.action === "stop" + ? "tool_runtime_slot.operator_stopped" + : "tool_runtime_slot.operator_restarted", + entityType: "tool_runtime_slot", + entityId: input.slotId, + details: { + runtimeKind: slot.runtimeKind, + status: slot.status, + slotKey: slot.slotKey, + }, + }); + return slot; + } catch (error) { + if (error instanceof ToolRuntimeSupervisorError) { + throw runtimeSupervisorHttpError(error); + } + throw error; + } + } + return { + ensureRuntimeSlot, + api: { + listRuntimeSlots: async (companyId: string): Promise => { + const rows = await db + .select() + .from(toolRuntimeSlots) + .where(eq(toolRuntimeSlots.companyId, companyId)) + .orderBy(desc(toolRuntimeSlots.updatedAt)); + return rows.map(toRuntimeSlot); + }, + + stopRuntimeSlot: ( + companyId: string, + slotId: string, + actor?: ActorInfo, + ): Promise => + controlRuntimeSlot({ companyId, slotId, action: "stop", actor }), + + restartRuntimeSlot: ( + companyId: string, + slotId: string, + actor?: ActorInfo, + ): Promise => + controlRuntimeSlot({ companyId, slotId, action: "restart", actor }) + }, + }; +}