From d1308bd3257b51e541a003123e82d83ea4dc0141 Mon Sep 17 00:00:00 2001 From: nguyenm7 <13559011+nguyenm7@users.noreply.github.com> Date: Fri, 18 Sep 2026 07:16:08 +0000 Subject: [PATCH] feat(telemetry): adopt connector telemetry events from the backend schema draft (DO NOT MERGE) Draft adoption of the three connector events proposed in #13582 and tracked in #13578. This change syncs the generated telemetry contract and promotes the proposed call sites. Provenance: - Backend schema draft: paperclipai/telemetry-backend PR #145, commit bf2709a0aa626b64fe1dd6c39235de394b507615. - Generated artifact: dist/client-sdk/paperclip-telemetry.ts from that exact commit, copied unmodified to packages/shared/src/telemetry/generated/paperclip-telemetry.ts per the documented manual copy path in the backend telemetry workflow. - Base: proposal branch head b7d358888396860d4e4806eec6042f63382fbabb. Changes: - Sync the generated telemetry contract (adds the 3 connector events). - Remove the three proposed-telemetry markers in events.ts; the wrappers now use the stable typed idiom (asEventDimension on enum dimensions). - Rename proposed-connector-events.test.ts to connector-events.test.ts and flip its assertions: the events are registered, queue, and send with exact dimensions; disabled-client and CI suppression stay proven. - Propose retention class operational_enum_count (90 days) for the three events in retention.ts, with tests. Verification: pnpm --filter @paperclipai/shared typecheck clean; shared telemetry suite 44/44; server connector-telemetry unit suite 20/20 (server code is unchanged; its tests mock both registration states). Co-Authored-By: Claude Fable 5 Co-Authored-By: Paperclip (cherry picked from commit 34101042e4e3842deec288877fc37cb9c984c306) --- ...vents.test.ts => connector-events.test.ts} | 67 ++++++---- packages/shared/src/telemetry/events.ts | 84 ++++++------- .../generated/paperclip-telemetry.ts | 116 ++++++++++++++++++ .../shared/src/telemetry/retention.test.ts | 6 + packages/shared/src/telemetry/retention.ts | 10 ++ 5 files changed, 216 insertions(+), 67 deletions(-) rename packages/shared/src/telemetry/{proposed-connector-events.test.ts => connector-events.test.ts} (66%) diff --git a/packages/shared/src/telemetry/proposed-connector-events.test.ts b/packages/shared/src/telemetry/connector-events.test.ts similarity index 66% rename from packages/shared/src/telemetry/proposed-connector-events.test.ts rename to packages/shared/src/telemetry/connector-events.test.ts index a8a6826e9de..e92c277bbc1 100644 --- a/packages/shared/src/telemetry/proposed-connector-events.test.ts +++ b/packages/shared/src/telemetry/connector-events.test.ts @@ -28,7 +28,7 @@ function makeClient(config?: { enabled?: boolean }) { }; } -function trackAllProposedConnectorEvents(client: TelemetryClient) { +function trackAllConnectorEvents(client: TelemetryClient) { trackConnectionCreated(client, { connector_key: "github", transport: "mcp_remote", @@ -56,29 +56,17 @@ function trackAllProposedConnectorEvents(client: TelemetryClient) { }); } -describe("proposed connector events against the real TelemetryClient", () => { +describe("registered connector events against the real TelemetryClient", () => { afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); }); - it("cannot queue or send: state stays untouched and nothing hits the wire", async () => { - vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); - const { client, stateFactory } = makeClient(); - - trackAllProposedConnectorEvents(client); - await client.flush(); - - expect(stateFactory).not.toHaveBeenCalled(); - expect(fetch).not.toHaveBeenCalled(); - }); - - it("differential control: a registered event through the same client does send", async () => { + it("queues and sends the three connector events with their exact dimensions", async () => { vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); const { client } = makeClient(); - trackAllProposedConnectorEvents(client); - client.track("project.created", {}); + trackAllConnectorEvents(client); await client.flush(); expect(fetch).toHaveBeenCalledTimes(1); @@ -86,23 +74,56 @@ describe("proposed connector events against the real TelemetryClient", () => { String((vi.mocked(fetch).mock.calls[0]?.[1] as RequestInit).body), ); expect(body.events).toEqual([ - expect.objectContaining({ name: "project.created" }), + expect.objectContaining({ + name: "connection.created", + dimensions: { + connector_key: "github", + transport: "mcp_remote", + auth_kind: "oauth", + setup_flow: "gallery", + status: "active", + enabled: true, + }, + }), + expect.objectContaining({ + name: "connection.updated", + dimensions: { + connector_key: "github", + transport: "mcp_remote", + auth_kind: "oauth", + change_source: "api", + previous_status: "draft", + status: "active", + previous_enabled: false, + enabled: true, + }, + }), + expect.objectContaining({ + name: "connection.invoked", + dimensions: { + connector_key: "github", + transport: "mcp_remote", + status: "succeeded", + origin: "agent", + duration_seconds: 3, + }, + }), ]); }); - it("reports the three connector event names as unregistered", () => { + it("reports the three connector event names as registered", () => { const { client } = makeClient(); - expect(client.isRegisteredEventName("connection.created")).toBe(false); - expect(client.isRegisteredEventName("connection.updated")).toBe(false); - expect(client.isRegisteredEventName("connection.invoked")).toBe(false); + expect(client.isRegisteredEventName("connection.created")).toBe(true); + expect(client.isRegisteredEventName("connection.updated")).toBe(true); + expect(client.isRegisteredEventName("connection.invoked")).toBe(true); expect(client.isRegisteredEventName("project.created")).toBe(true); }); - it("a disabled client drops even registered events", async () => { + it("a disabled client drops connector events", async () => { vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true })); const { client, stateFactory } = makeClient({ enabled: false }); - client.track("project.created", {}); + trackAllConnectorEvents(client); await client.flush(); expect(stateFactory).not.toHaveBeenCalled(); diff --git a/packages/shared/src/telemetry/events.ts b/packages/shared/src/telemetry/events.ts index 71f78bb2f3f..7039f186d11 100644 --- a/packages/shared/src/telemetry/events.ts +++ b/packages/shared/src/telemetry/events.ts @@ -156,70 +156,66 @@ export function trackAgentTaskRun( export function trackConnectionCreated( client: TelemetryClient, dims: { - connector_key: RawDimension<"custom">; - transport: RawDimension<"mcp_remote" | "rest_api" | "local_stdio">; - auth_kind: RawDimension<"oauth" | "api_key" | "none">; - setup_flow: RawDimension<"gallery" | "api" | "example">; - status: RawDimension<"draft" | "active" | "disabled" | "archived">; + connector_key: string; + transport: RawDimension; + auth_kind: RawDimension; + setup_flow: RawDimension; + status: RawDimension; enabled: boolean; }, ): void { - client.track( - // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/13578): measure which catalog connectors installations create connections for - "connection.created", - dims, - ); + client.track("connection.created", { + connector_key: dims.connector_key, + transport: asEventDimension(dims.transport), + auth_kind: asEventDimension(dims.auth_kind), + setup_flow: asEventDimension(dims.setup_flow), + status: asEventDimension(dims.status), + enabled: dims.enabled, + }); } export function trackConnectionUpdated( client: TelemetryClient, dims: { - connector_key: RawDimension<"custom">; - transport: RawDimension<"mcp_remote" | "rest_api" | "local_stdio">; - auth_kind: RawDimension<"oauth" | "api_key" | "none">; - change_source: RawDimension< - | "api" - | "gallery" - | "oauth_callback" - | "credential_refresh" - | "archive" - | "example" - >; - previous_status: RawDimension<"draft" | "active" | "disabled" | "archived">; - status: RawDimension<"draft" | "active" | "disabled" | "archived">; + connector_key: string; + transport: RawDimension; + auth_kind: RawDimension; + change_source: RawDimension; + previous_status: RawDimension; + status: RawDimension; previous_enabled: boolean; enabled: boolean; }, ): void { - client.track( - // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/13578): measure connector lifecycle transitions (configured, paused, archived) after creation - "connection.updated", - dims, - ); + client.track("connection.updated", { + connector_key: dims.connector_key, + transport: asEventDimension(dims.transport), + auth_kind: asEventDimension(dims.auth_kind), + change_source: asEventDimension(dims.change_source), + previous_status: asEventDimension(dims.previous_status), + status: asEventDimension(dims.status), + previous_enabled: dims.previous_enabled, + enabled: dims.enabled, + }); } export function trackConnectionInvoked( client: TelemetryClient, dims: { - connector_key: RawDimension<"custom">; - transport: RawDimension<"mcp_remote" | "rest_api" | "local_stdio">; - status: RawDimension< - | "succeeded" - | "failed" - | "denied" - | "cancelled" - | "timed_out" - | "rate_limited" - >; - origin: RawDimension<"setup_test" | "agent" | "user" | "system" | "plugin">; + connector_key: string; + transport: RawDimension; + status: RawDimension; + origin: RawDimension; duration_seconds?: number; }, ): void { - client.track( - // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/13578): measure whether connected connectors are successfully used and where invocations fail; records completed invocation attempts and their terminal status, never invocation starts - "connection.invoked", - dims, - ); + client.track("connection.invoked", { + connector_key: dims.connector_key, + transport: asEventDimension(dims.transport), + status: asEventDimension(dims.status), + origin: asEventDimension(dims.origin), + ...(dims.duration_seconds === undefined ? {} : { duration_seconds: dims.duration_seconds }), + }); } export function trackErrorHandlerCrash( diff --git a/packages/shared/src/telemetry/generated/paperclip-telemetry.ts b/packages/shared/src/telemetry/generated/paperclip-telemetry.ts index b0efe356e7d..b552647ce80 100644 --- a/packages/shared/src/telemetry/generated/paperclip-telemetry.ts +++ b/packages/shared/src/telemetry/generated/paperclip-telemetry.ts @@ -37,6 +37,34 @@ source_ref?: string source_ref_hashed?: boolean } +export interface PaperclipConnectorConnectionCreatedDimensions { +auth_kind: ("oauth" | "api_key" | "none") +connector_key: string +enabled: boolean +setup_flow: ("gallery" | "api" | "example" | "composio_sync") +status: ("draft" | "active" | "disabled" | "archived") +transport: ("mcp_remote" | "rest_api" | "local_stdio") +} + +export interface PaperclipConnectorConnectionUpdatedDimensions { +auth_kind: ("oauth" | "api_key" | "none") +change_source: ("update_api" | "gallery_setup" | "oauth_callback" | "credential_refresh" | "archive" | "example" | "composio_sync") +connector_key: string +enabled: boolean +previous_enabled: boolean +previous_status: ("draft" | "active" | "disabled" | "archived") +status: ("draft" | "active" | "disabled" | "archived") +transport: ("mcp_remote" | "rest_api" | "local_stdio") +} + +export interface PaperclipConnectorInvocationCompletedDimensions { +connector_key: string +duration_seconds?: number +origin: ("setup_test" | "agent" | "user" | "system" | "plugin") +status: ("succeeded" | "failed" | "denied" | "cancelled" | "timed_out" | "rate_limited") +transport: ("mcp_remote" | "rest_api" | "local_stdio") +} + export interface PaperclipErrorHandlerCrashDimensions { error_code: string } @@ -105,6 +133,9 @@ export type PaperclipEventName = | "agent.task_completed" | "agent.task_run" | "company.imported" + | "connector.connection_created" + | "connector.connection_updated" + | "connector.invocation_completed" | "error.handler_crash" | "goal.created" | "install.completed" @@ -122,6 +153,9 @@ export interface EventDimensionsMap { "agent.task_completed": PaperclipAgentTaskCompletedDimensions; "agent.task_run": PaperclipAgentTaskRunDimensions; "company.imported": PaperclipCompanyImportedDimensions; + "connector.connection_created": PaperclipConnectorConnectionCreatedDimensions; + "connector.connection_updated": PaperclipConnectorConnectionUpdatedDimensions; + "connector.invocation_completed": PaperclipConnectorInvocationCompletedDimensions; "error.handler_crash": PaperclipErrorHandlerCrashDimensions; "goal.created": PaperclipGoalCreatedDimensions; "install.completed": PaperclipInstallCompletedDimensions; @@ -140,6 +174,9 @@ export const PAPERCLIP_EVENTS = { "agent.task_completed": "agent.task_completed", "agent.task_run": "agent.task_run", "company.imported": "company.imported", + "connector.connection_created": "connector.connection_created", + "connector.connection_updated": "connector.connection_updated", + "connector.invocation_completed": "connector.invocation_completed", "error.handler_crash": "error.handler_crash", "goal.created": "goal.created", "install.completed": "install.completed", @@ -274,6 +311,85 @@ export const PAPERCLIP_ENUM_DESCRIPTIONS = { "unknown": "Source type could not be classified." } }, + "connector.connection_created": { + "auth_kind": { + "oauth": "Connection authenticates with an OAuth flow.", + "api_key": "Connection authenticates with a stored API key.", + "none": "Connection requires no authentication." + }, + "setup_flow": { + "gallery": "Connection was created through the app-gallery setup flow.", + "api": "Connection was created through the connections API.", + "example": "Connection was created or upserted as a bundled example connection.", + "composio_sync": "Connection was created by syncing a Composio child connection." + }, + "status": { + "draft": "Connection row exists but setup has not produced a usable configuration yet.", + "active": "Connection is fully configured and available for use.", + "disabled": "Connection is disabled and excluded from tool use.", + "archived": "Connection is archived and excluded from tool use." + }, + "transport": { + "mcp_remote": "Connection talks to a remote MCP server over HTTP.", + "rest_api": "Connection calls the provider's REST API directly.", + "local_stdio": "Connection runs a local MCP server over stdio." + } + }, + "connector.connection_updated": { + "auth_kind": { + "oauth": "Connection authenticates with an OAuth flow.", + "api_key": "Connection authenticates with a stored API key.", + "none": "Connection requires no authentication." + }, + "change_source": { + "update_api": "A connections API update changed the persisted lifecycle state.", + "gallery_setup": "An app-gallery setup step changed the persisted lifecycle state.", + "oauth_callback": "An OAuth callback completion changed the persisted lifecycle state.", + "credential_refresh": "A credential refresh changed the persisted lifecycle state.", + "archive": "An archive operation changed the persisted lifecycle state.", + "example": "An example-connection upsert changed the persisted lifecycle state.", + "composio_sync": "A Composio child-connection sync changed the persisted lifecycle state." + }, + "previous_status": { + "draft": "Connection row exists but setup has not produced a usable configuration yet.", + "active": "Connection is fully configured and available for use.", + "disabled": "Connection is disabled and excluded from tool use.", + "archived": "Connection is archived and excluded from tool use." + }, + "status": { + "draft": "Connection row exists but setup has not produced a usable configuration yet.", + "active": "Connection is fully configured and available for use.", + "disabled": "Connection is disabled and excluded from tool use.", + "archived": "Connection is archived and excluded from tool use." + }, + "transport": { + "mcp_remote": "Connection talks to a remote MCP server over HTTP.", + "rest_api": "Connection calls the provider's REST API directly.", + "local_stdio": "Connection runs a local MCP server over stdio." + } + }, + "connector.invocation_completed": { + "origin": { + "setup_test": "A human ran the connection's setup test, identified by durable invocation columns: user actor with no run, issue, or gateway reference and a connection id.", + "agent": "An agent run invoked the tool.", + "user": "A human invoked the tool outside the setup-test shape.", + "system": "A system-initiated process invoked the tool.", + "plugin": "A plugin invoked the tool." + }, + "status": { + "succeeded": "The tool invocation completed successfully.", + "failed": "The tool invocation reached a terminal failure.", + "denied": "The tool invocation was denied by policy or by a human approval decision.", + "cancelled": "The tool invocation was cancelled before completion. Declared in the application status enum; no current application code path writes this value.", + "timed_out": "The tool invocation exceeded its time limit.", + "rate_limited": "The tool invocation was rejected by rate limiting." + }, + "transport": { + "mcp_remote": "Connection talks to a remote MCP server over HTTP.", + "rest_api": "Connection calls the provider's REST API directly.", + "local_stdio": "Connection runs a local MCP server over stdio." + } + }, "goal.created": { "goal_level": { "company": "Goal applies at company scope.", diff --git a/packages/shared/src/telemetry/retention.test.ts b/packages/shared/src/telemetry/retention.test.ts index 9d54db1371b..24928a49d73 100644 --- a/packages/shared/src/telemetry/retention.test.ts +++ b/packages/shared/src/telemetry/retention.test.ts @@ -23,4 +23,10 @@ describe("telemetry retention contract", () => { expect(EVENT_RETENTION_CLASS["interaction.created"]).toBe("operational_enum_count"); expect(EVENT_RETENTION_CLASS["interaction.resolved"]).toBe("operational_enum_count"); }); + + it("connector telemetry uses enum/count retention", () => { + expect(EVENT_RETENTION_CLASS["connector.connection_created"]).toBe("operational_enum_count"); + expect(EVENT_RETENTION_CLASS["connector.connection_updated"]).toBe("operational_enum_count"); + expect(EVENT_RETENTION_CLASS["connector.invocation_completed"]).toBe("operational_enum_count"); + }); }); diff --git a/packages/shared/src/telemetry/retention.ts b/packages/shared/src/telemetry/retention.ts index 543e1085298..45231d3be74 100644 --- a/packages/shared/src/telemetry/retention.ts +++ b/packages/shared/src/telemetry/retention.ts @@ -42,9 +42,19 @@ export type RetentionClass = keyof typeof RETENTION_DAYS; * counts (rotations detected). No token material and no PII. * External-system event (Codex CLI) — not yet in PaperclipEventName. * Class: operational_enum_count — 90-day window. + * + * connector.connection_created / connector.connection_updated / + * connector.invocation_completed + * Carry only closed enums, booleans, a coarse rounded-seconds duration, and + * the reviewed catalog slug (or the literal "custom") as connector_key. No + * token material and no PII. + * Class: operational_enum_count — 90-day window. */ export const EVENT_RETENTION_CLASS: Partial> = { "codex.credential_health": "operational_enum_count", "interaction.created": "operational_enum_count", "interaction.resolved": "operational_enum_count", + "connector.connection_created": "operational_enum_count", + "connector.connection_updated": "operational_enum_count", + "connector.invocation_completed": "operational_enum_count", };