mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-25 02:26:12 +02:00
feat(telemetry): adopt connector telemetry events from the backend schema draft (DO NOT MERGE)
Draft adoption of the three connector events proposed in #13582 and tracked in #13578. This change syncs the generated telemetry contract and promotes the proposed call sites. Provenance: - Backend schema draft: paperclipai/telemetry-backend PR #145, commit bf2709a0aa626b64fe1dd6c39235de394b507615. - Generated artifact: dist/client-sdk/paperclip-telemetry.ts from that exact commit, copied unmodified to packages/shared/src/telemetry/generated/paperclip-telemetry.ts per the documented manual copy path in the backend telemetry workflow. - Base: proposal branch head b7d358888396860d4e4806eec6042f63382fbabb. Changes: - Sync the generated telemetry contract (adds the 3 connector events). - Remove the three proposed-telemetry markers in events.ts; the wrappers now use the stable typed idiom (asEventDimension on enum dimensions). - Rename proposed-connector-events.test.ts to connector-events.test.ts and flip its assertions: the events are registered, queue, and send with exact dimensions; disabled-client and CI suppression stay proven. - Propose retention class operational_enum_count (90 days) for the three events in retention.ts, with tests. Verification: pnpm --filter @paperclipai/shared typecheck clean; shared telemetry suite 44/44; server connector-telemetry unit suite 20/20 (server code is unchanged; its tests mock both registration states). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Co-Authored-By: Paperclip <noreply@paperclip.ing> (cherry picked from commit 34101042e4e3842deec288877fc37cb9c984c306)
This commit is contained in:
1 parent
30182c704c
commit
d1308bd325
5 files changed
+216
-67
No files matched your search
+44
-23
@@ -28,7 +28,7 @@ function makeClient(config?: { enabled?: boolean }) {
|
||||
};
|
||||
}
|
||||
|
||||
function trackAllProposedConnectorEvents(client: TelemetryClient) {
|
||||
function trackAllConnectorEvents(client: TelemetryClient) {
|
||||
trackConnectionCreated(client, {
|
||||
connector_key: "github",
|
||||
transport: "mcp_remote",
|
||||
@@ -56,29 +56,17 @@ function trackAllProposedConnectorEvents(client: TelemetryClient) {
|
||||
});
|
||||
}
|
||||
|
||||
describe("proposed connector events against the real TelemetryClient", () => {
|
||||
describe("registered connector events against the real TelemetryClient", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("cannot queue or send: state stays untouched and nothing hits the wire", async () => {
|
||||
vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true }));
|
||||
const { client, stateFactory } = makeClient();
|
||||
|
||||
trackAllProposedConnectorEvents(client);
|
||||
await client.flush();
|
||||
|
||||
expect(stateFactory).not.toHaveBeenCalled();
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("differential control: a registered event through the same client does send", async () => {
|
||||
it("queues and sends the three connector events with their exact dimensions", async () => {
|
||||
vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true }));
|
||||
const { client } = makeClient();
|
||||
|
||||
trackAllProposedConnectorEvents(client);
|
||||
client.track("project.created", {});
|
||||
trackAllConnectorEvents(client);
|
||||
await client.flush();
|
||||
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
@@ -86,23 +74,56 @@ describe("proposed connector events against the real TelemetryClient", () => {
|
||||
String((vi.mocked(fetch).mock.calls[0]?.[1] as RequestInit).body),
|
||||
);
|
||||
expect(body.events).toEqual([
|
||||
expect.objectContaining({ name: "project.created" }),
|
||||
expect.objectContaining({
|
||||
name: "connection.created",
|
||||
dimensions: {
|
||||
connector_key: "github",
|
||||
transport: "mcp_remote",
|
||||
auth_kind: "oauth",
|
||||
setup_flow: "gallery",
|
||||
status: "active",
|
||||
enabled: true,
|
||||
},
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "connection.updated",
|
||||
dimensions: {
|
||||
connector_key: "github",
|
||||
transport: "mcp_remote",
|
||||
auth_kind: "oauth",
|
||||
change_source: "api",
|
||||
previous_status: "draft",
|
||||
status: "active",
|
||||
previous_enabled: false,
|
||||
enabled: true,
|
||||
},
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "connection.invoked",
|
||||
dimensions: {
|
||||
connector_key: "github",
|
||||
transport: "mcp_remote",
|
||||
status: "succeeded",
|
||||
origin: "agent",
|
||||
duration_seconds: 3,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports the three connector event names as unregistered", () => {
|
||||
it("reports the three connector event names as registered", () => {
|
||||
const { client } = makeClient();
|
||||
expect(client.isRegisteredEventName("connection.created")).toBe(false);
|
||||
expect(client.isRegisteredEventName("connection.updated")).toBe(false);
|
||||
expect(client.isRegisteredEventName("connection.invoked")).toBe(false);
|
||||
expect(client.isRegisteredEventName("connection.created")).toBe(true);
|
||||
expect(client.isRegisteredEventName("connection.updated")).toBe(true);
|
||||
expect(client.isRegisteredEventName("connection.invoked")).toBe(true);
|
||||
expect(client.isRegisteredEventName("project.created")).toBe(true);
|
||||
});
|
||||
|
||||
it("a disabled client drops even registered events", async () => {
|
||||
it("a disabled client drops connector events", async () => {
|
||||
vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true }));
|
||||
const { client, stateFactory } = makeClient({ enabled: false });
|
||||
|
||||
client.track("project.created", {});
|
||||
trackAllConnectorEvents(client);
|
||||
await client.flush();
|
||||
|
||||
expect(stateFactory).not.toHaveBeenCalled();
|
||||
@@ -156,70 +156,66 @@ export function trackAgentTaskRun(
|
||||
export function trackConnectionCreated(
|
||||
client: TelemetryClient,
|
||||
dims: {
|
||||
connector_key: RawDimension<"custom">;
|
||||
transport: RawDimension<"mcp_remote" | "rest_api" | "local_stdio">;
|
||||
auth_kind: RawDimension<"oauth" | "api_key" | "none">;
|
||||
setup_flow: RawDimension<"gallery" | "api" | "example">;
|
||||
status: RawDimension<"draft" | "active" | "disabled" | "archived">;
|
||||
connector_key: string;
|
||||
transport: RawDimension<EventDimensionsMap["connection.created"]["transport"]>;
|
||||
auth_kind: RawDimension<EventDimensionsMap["connection.created"]["auth_kind"]>;
|
||||
setup_flow: RawDimension<EventDimensionsMap["connection.created"]["setup_flow"]>;
|
||||
status: RawDimension<EventDimensionsMap["connection.created"]["status"]>;
|
||||
enabled: boolean;
|
||||
},
|
||||
): void {
|
||||
client.track(
|
||||
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/13578): measure which catalog connectors installations create connections for
|
||||
"connection.created",
|
||||
dims,
|
||||
);
|
||||
client.track("connection.created", {
|
||||
connector_key: dims.connector_key,
|
||||
transport: asEventDimension(dims.transport),
|
||||
auth_kind: asEventDimension(dims.auth_kind),
|
||||
setup_flow: asEventDimension(dims.setup_flow),
|
||||
status: asEventDimension(dims.status),
|
||||
enabled: dims.enabled,
|
||||
});
|
||||
}
|
||||
|
||||
export function trackConnectionUpdated(
|
||||
client: TelemetryClient,
|
||||
dims: {
|
||||
connector_key: RawDimension<"custom">;
|
||||
transport: RawDimension<"mcp_remote" | "rest_api" | "local_stdio">;
|
||||
auth_kind: RawDimension<"oauth" | "api_key" | "none">;
|
||||
change_source: RawDimension<
|
||||
| "api"
|
||||
| "gallery"
|
||||
| "oauth_callback"
|
||||
| "credential_refresh"
|
||||
| "archive"
|
||||
| "example"
|
||||
>;
|
||||
previous_status: RawDimension<"draft" | "active" | "disabled" | "archived">;
|
||||
status: RawDimension<"draft" | "active" | "disabled" | "archived">;
|
||||
connector_key: string;
|
||||
transport: RawDimension<EventDimensionsMap["connection.updated"]["transport"]>;
|
||||
auth_kind: RawDimension<EventDimensionsMap["connection.updated"]["auth_kind"]>;
|
||||
change_source: RawDimension<EventDimensionsMap["connection.updated"]["change_source"]>;
|
||||
previous_status: RawDimension<EventDimensionsMap["connection.updated"]["previous_status"]>;
|
||||
status: RawDimension<EventDimensionsMap["connection.updated"]["status"]>;
|
||||
previous_enabled: boolean;
|
||||
enabled: boolean;
|
||||
},
|
||||
): void {
|
||||
client.track(
|
||||
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/13578): measure connector lifecycle transitions (configured, paused, archived) after creation
|
||||
"connection.updated",
|
||||
dims,
|
||||
);
|
||||
client.track("connection.updated", {
|
||||
connector_key: dims.connector_key,
|
||||
transport: asEventDimension(dims.transport),
|
||||
auth_kind: asEventDimension(dims.auth_kind),
|
||||
change_source: asEventDimension(dims.change_source),
|
||||
previous_status: asEventDimension(dims.previous_status),
|
||||
status: asEventDimension(dims.status),
|
||||
previous_enabled: dims.previous_enabled,
|
||||
enabled: dims.enabled,
|
||||
});
|
||||
}
|
||||
|
||||
export function trackConnectionInvoked(
|
||||
client: TelemetryClient,
|
||||
dims: {
|
||||
connector_key: RawDimension<"custom">;
|
||||
transport: RawDimension<"mcp_remote" | "rest_api" | "local_stdio">;
|
||||
status: RawDimension<
|
||||
| "succeeded"
|
||||
| "failed"
|
||||
| "denied"
|
||||
| "cancelled"
|
||||
| "timed_out"
|
||||
| "rate_limited"
|
||||
>;
|
||||
origin: RawDimension<"setup_test" | "agent" | "user" | "system" | "plugin">;
|
||||
connector_key: string;
|
||||
transport: RawDimension<EventDimensionsMap["connection.invoked"]["transport"]>;
|
||||
status: RawDimension<EventDimensionsMap["connection.invoked"]["status"]>;
|
||||
origin: RawDimension<EventDimensionsMap["connection.invoked"]["origin"]>;
|
||||
duration_seconds?: number;
|
||||
},
|
||||
): void {
|
||||
client.track(
|
||||
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/13578): measure whether connected connectors are successfully used and where invocations fail; records completed invocation attempts and their terminal status, never invocation starts
|
||||
"connection.invoked",
|
||||
dims,
|
||||
);
|
||||
client.track("connection.invoked", {
|
||||
connector_key: dims.connector_key,
|
||||
transport: asEventDimension(dims.transport),
|
||||
status: asEventDimension(dims.status),
|
||||
origin: asEventDimension(dims.origin),
|
||||
...(dims.duration_seconds === undefined ? {} : { duration_seconds: dims.duration_seconds }),
|
||||
});
|
||||
}
|
||||
|
||||
export function trackErrorHandlerCrash(
|
||||
|
||||
@@ -37,6 +37,34 @@ source_ref?: string
|
||||
source_ref_hashed?: boolean
|
||||
}
|
||||
|
||||
export interface PaperclipConnectorConnectionCreatedDimensions {
|
||||
auth_kind: ("oauth" | "api_key" | "none")
|
||||
connector_key: string
|
||||
enabled: boolean
|
||||
setup_flow: ("gallery" | "api" | "example" | "composio_sync")
|
||||
status: ("draft" | "active" | "disabled" | "archived")
|
||||
transport: ("mcp_remote" | "rest_api" | "local_stdio")
|
||||
}
|
||||
|
||||
export interface PaperclipConnectorConnectionUpdatedDimensions {
|
||||
auth_kind: ("oauth" | "api_key" | "none")
|
||||
change_source: ("update_api" | "gallery_setup" | "oauth_callback" | "credential_refresh" | "archive" | "example" | "composio_sync")
|
||||
connector_key: string
|
||||
enabled: boolean
|
||||
previous_enabled: boolean
|
||||
previous_status: ("draft" | "active" | "disabled" | "archived")
|
||||
status: ("draft" | "active" | "disabled" | "archived")
|
||||
transport: ("mcp_remote" | "rest_api" | "local_stdio")
|
||||
}
|
||||
|
||||
export interface PaperclipConnectorInvocationCompletedDimensions {
|
||||
connector_key: string
|
||||
duration_seconds?: number
|
||||
origin: ("setup_test" | "agent" | "user" | "system" | "plugin")
|
||||
status: ("succeeded" | "failed" | "denied" | "cancelled" | "timed_out" | "rate_limited")
|
||||
transport: ("mcp_remote" | "rest_api" | "local_stdio")
|
||||
}
|
||||
|
||||
export interface PaperclipErrorHandlerCrashDimensions {
|
||||
error_code: string
|
||||
}
|
||||
@@ -105,6 +133,9 @@ export type PaperclipEventName =
|
||||
| "agent.task_completed"
|
||||
| "agent.task_run"
|
||||
| "company.imported"
|
||||
| "connector.connection_created"
|
||||
| "connector.connection_updated"
|
||||
| "connector.invocation_completed"
|
||||
| "error.handler_crash"
|
||||
| "goal.created"
|
||||
| "install.completed"
|
||||
@@ -122,6 +153,9 @@ export interface EventDimensionsMap {
|
||||
"agent.task_completed": PaperclipAgentTaskCompletedDimensions;
|
||||
"agent.task_run": PaperclipAgentTaskRunDimensions;
|
||||
"company.imported": PaperclipCompanyImportedDimensions;
|
||||
"connector.connection_created": PaperclipConnectorConnectionCreatedDimensions;
|
||||
"connector.connection_updated": PaperclipConnectorConnectionUpdatedDimensions;
|
||||
"connector.invocation_completed": PaperclipConnectorInvocationCompletedDimensions;
|
||||
"error.handler_crash": PaperclipErrorHandlerCrashDimensions;
|
||||
"goal.created": PaperclipGoalCreatedDimensions;
|
||||
"install.completed": PaperclipInstallCompletedDimensions;
|
||||
@@ -140,6 +174,9 @@ export const PAPERCLIP_EVENTS = {
|
||||
"agent.task_completed": "agent.task_completed",
|
||||
"agent.task_run": "agent.task_run",
|
||||
"company.imported": "company.imported",
|
||||
"connector.connection_created": "connector.connection_created",
|
||||
"connector.connection_updated": "connector.connection_updated",
|
||||
"connector.invocation_completed": "connector.invocation_completed",
|
||||
"error.handler_crash": "error.handler_crash",
|
||||
"goal.created": "goal.created",
|
||||
"install.completed": "install.completed",
|
||||
@@ -274,6 +311,85 @@ export const PAPERCLIP_ENUM_DESCRIPTIONS = {
|
||||
"unknown": "Source type could not be classified."
|
||||
}
|
||||
},
|
||||
"connector.connection_created": {
|
||||
"auth_kind": {
|
||||
"oauth": "Connection authenticates with an OAuth flow.",
|
||||
"api_key": "Connection authenticates with a stored API key.",
|
||||
"none": "Connection requires no authentication."
|
||||
},
|
||||
"setup_flow": {
|
||||
"gallery": "Connection was created through the app-gallery setup flow.",
|
||||
"api": "Connection was created through the connections API.",
|
||||
"example": "Connection was created or upserted as a bundled example connection.",
|
||||
"composio_sync": "Connection was created by syncing a Composio child connection."
|
||||
},
|
||||
"status": {
|
||||
"draft": "Connection row exists but setup has not produced a usable configuration yet.",
|
||||
"active": "Connection is fully configured and available for use.",
|
||||
"disabled": "Connection is disabled and excluded from tool use.",
|
||||
"archived": "Connection is archived and excluded from tool use."
|
||||
},
|
||||
"transport": {
|
||||
"mcp_remote": "Connection talks to a remote MCP server over HTTP.",
|
||||
"rest_api": "Connection calls the provider's REST API directly.",
|
||||
"local_stdio": "Connection runs a local MCP server over stdio."
|
||||
}
|
||||
},
|
||||
"connector.connection_updated": {
|
||||
"auth_kind": {
|
||||
"oauth": "Connection authenticates with an OAuth flow.",
|
||||
"api_key": "Connection authenticates with a stored API key.",
|
||||
"none": "Connection requires no authentication."
|
||||
},
|
||||
"change_source": {
|
||||
"update_api": "A connections API update changed the persisted lifecycle state.",
|
||||
"gallery_setup": "An app-gallery setup step changed the persisted lifecycle state.",
|
||||
"oauth_callback": "An OAuth callback completion changed the persisted lifecycle state.",
|
||||
"credential_refresh": "A credential refresh changed the persisted lifecycle state.",
|
||||
"archive": "An archive operation changed the persisted lifecycle state.",
|
||||
"example": "An example-connection upsert changed the persisted lifecycle state.",
|
||||
"composio_sync": "A Composio child-connection sync changed the persisted lifecycle state."
|
||||
},
|
||||
"previous_status": {
|
||||
"draft": "Connection row exists but setup has not produced a usable configuration yet.",
|
||||
"active": "Connection is fully configured and available for use.",
|
||||
"disabled": "Connection is disabled and excluded from tool use.",
|
||||
"archived": "Connection is archived and excluded from tool use."
|
||||
},
|
||||
"status": {
|
||||
"draft": "Connection row exists but setup has not produced a usable configuration yet.",
|
||||
"active": "Connection is fully configured and available for use.",
|
||||
"disabled": "Connection is disabled and excluded from tool use.",
|
||||
"archived": "Connection is archived and excluded from tool use."
|
||||
},
|
||||
"transport": {
|
||||
"mcp_remote": "Connection talks to a remote MCP server over HTTP.",
|
||||
"rest_api": "Connection calls the provider's REST API directly.",
|
||||
"local_stdio": "Connection runs a local MCP server over stdio."
|
||||
}
|
||||
},
|
||||
"connector.invocation_completed": {
|
||||
"origin": {
|
||||
"setup_test": "A human ran the connection's setup test, identified by durable invocation columns: user actor with no run, issue, or gateway reference and a connection id.",
|
||||
"agent": "An agent run invoked the tool.",
|
||||
"user": "A human invoked the tool outside the setup-test shape.",
|
||||
"system": "A system-initiated process invoked the tool.",
|
||||
"plugin": "A plugin invoked the tool."
|
||||
},
|
||||
"status": {
|
||||
"succeeded": "The tool invocation completed successfully.",
|
||||
"failed": "The tool invocation reached a terminal failure.",
|
||||
"denied": "The tool invocation was denied by policy or by a human approval decision.",
|
||||
"cancelled": "The tool invocation was cancelled before completion. Declared in the application status enum; no current application code path writes this value.",
|
||||
"timed_out": "The tool invocation exceeded its time limit.",
|
||||
"rate_limited": "The tool invocation was rejected by rate limiting."
|
||||
},
|
||||
"transport": {
|
||||
"mcp_remote": "Connection talks to a remote MCP server over HTTP.",
|
||||
"rest_api": "Connection calls the provider's REST API directly.",
|
||||
"local_stdio": "Connection runs a local MCP server over stdio."
|
||||
}
|
||||
},
|
||||
"goal.created": {
|
||||
"goal_level": {
|
||||
"company": "Goal applies at company scope.",
|
||||
|
||||
@@ -23,4 +23,10 @@ describe("telemetry retention contract", () => {
|
||||
expect(EVENT_RETENTION_CLASS["interaction.created"]).toBe("operational_enum_count");
|
||||
expect(EVENT_RETENTION_CLASS["interaction.resolved"]).toBe("operational_enum_count");
|
||||
});
|
||||
|
||||
it("connector telemetry uses enum/count retention", () => {
|
||||
expect(EVENT_RETENTION_CLASS["connector.connection_created"]).toBe("operational_enum_count");
|
||||
expect(EVENT_RETENTION_CLASS["connector.connection_updated"]).toBe("operational_enum_count");
|
||||
expect(EVENT_RETENTION_CLASS["connector.invocation_completed"]).toBe("operational_enum_count");
|
||||
});
|
||||
});
|
||||
@@ -42,9 +42,19 @@ export type RetentionClass = keyof typeof RETENTION_DAYS;
|
||||
* counts (rotations detected). No token material and no PII.
|
||||
* External-system event (Codex CLI) — not yet in PaperclipEventName.
|
||||
* Class: operational_enum_count — 90-day window.
|
||||
*
|
||||
* connector.connection_created / connector.connection_updated /
|
||||
* connector.invocation_completed
|
||||
* Carry only closed enums, booleans, a coarse rounded-seconds duration, and
|
||||
* the reviewed catalog slug (or the literal "custom") as connector_key. No
|
||||
* token material and no PII.
|
||||
* Class: operational_enum_count — 90-day window.
|
||||
*/
|
||||
export const EVENT_RETENTION_CLASS: Partial<Record<string, RetentionClass>> = {
|
||||
"codex.credential_health": "operational_enum_count",
|
||||
"interaction.created": "operational_enum_count",
|
||||
"interaction.resolved": "operational_enum_count",
|
||||
"connector.connection_created": "operational_enum_count",
|
||||
"connector.connection_updated": "operational_enum_count",
|
||||
"connector.invocation_completed": "operational_enum_count",
|
||||
};
|
||||
Reference in new issue
Block a user