diff --git a/packages/shared/src/telemetry/client.test.ts b/packages/shared/src/telemetry/client.test.ts index 0de47d42220..d1c6c540797 100644 --- a/packages/shared/src/telemetry/client.test.ts +++ b/packages/shared/src/telemetry/client.test.ts @@ -58,32 +58,32 @@ describe("TelemetryClient runtime event gate", () => { const { client, stateFactory } = makeClient(); trackSkillCreated(client, { - skill_id: "11111111-1111-4111-8111-111111111111", - creation_source: "blank", - sharing_scope: "company", - category_count: 1, - file_count: 2, + skillId: "11111111-1111-4111-8111-111111111111", + creationSource: "blank", + sharingScope: "company", + categoryCount: 1, + fileCount: 2, }); trackSkillVersionSaved(client, { - skill_id: "11111111-1111-4111-8111-111111111111", - revision_number: 2, - file_type: "skill", + skillId: "11111111-1111-4111-8111-111111111111", + revisionNumber: 2, + fileType: "skill", }); trackSkillTestRun(client, { - skill_id: "11111111-1111-4111-8111-111111111111", + skillId: "11111111-1111-4111-8111-111111111111", status: "queued", - run_source: "run", - ad_hoc: true, - template_used: false, + runSource: "run", + adHoc: true, + templateUsed: false, }); trackSkillForked(client, { - skill_id: "22222222-2222-4222-8222-222222222222", - fork_from_skill_id: "11111111-1111-4111-8111-111111111111", - source_type: "catalog", - sharing_scope: "private", - reassign_agent_count: 0, + skillId: "22222222-2222-4222-8222-222222222222", + forkFromSkillId: "11111111-1111-4111-8111-111111111111", + sourceType: "catalog", + sharingScope: "private", + reassignAgentCount: 0, }); - trackSkillShareLinkCopied(client, { sharing_scope: "public_link" }); + trackSkillShareLinkCopied(client, { sharingScope: "public_link" }); await client.flush(); diff --git a/packages/shared/src/telemetry/events.ts b/packages/shared/src/telemetry/events.ts index 49e672b4d3b..b596637d7d8 100644 --- a/packages/shared/src/telemetry/events.ts +++ b/packages/shared/src/telemetry/events.ts @@ -95,22 +95,22 @@ export function trackSkillImported( export function trackSkillCreated( client: TelemetryClient, dims: { - skill_id: string; - creation_source: RawDimension<"blank" | "fork" | "project_scan">; - sharing_scope: RawDimension<"private" | "company" | "public_link">; - category_count: number; - file_count: number; + skillId: string; + creationSource: RawDimension<"blank" | "fork" | "project_scan">; + sharingScope: RawDimension<"private" | "company" | "public_link">; + categoryCount: number; + fileCount: number; }, ): void { client.track( // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): measure successful Skill Studio skill creation funnel "skill.created", { - skill_id: dims.skill_id, - creation_source: dims.creation_source, - sharing_scope: dims.sharing_scope, - category_count: dims.category_count, - file_count: dims.file_count, + skill_id: dims.skillId, + creation_source: asEventDimension(dims.creationSource), + sharing_scope: asEventDimension(dims.sharingScope), + category_count: dims.categoryCount, + file_count: dims.fileCount, }, ); } @@ -118,18 +118,18 @@ export function trackSkillCreated( export function trackSkillVersionSaved( client: TelemetryClient, dims: { - skill_id: string; - revision_number: number; - file_type: RawDimension<"skill" | "markdown" | "reference" | "script" | "asset" | "other">; + skillId: string; + revisionNumber: number; + fileType: RawDimension<"skill" | "markdown" | "reference" | "script" | "asset" | "other">; }, ): void { client.track( // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): measure Skill Studio editor save and version creation usage "skill.version_saved", { - skill_id: dims.skill_id, - revision_number: dims.revision_number, - file_type: dims.file_type, + skill_id: dims.skillId, + revision_number: dims.revisionNumber, + file_type: asEventDimension(dims.fileType), }, ); } @@ -137,22 +137,22 @@ export function trackSkillVersionSaved( export function trackSkillTestRun( client: TelemetryClient, dims: { - skill_id: string; + skillId: string; status: RawDimension<"queued" | "running" | "succeeded" | "failed" | "cancelled">; - run_source: RawDimension<"run" | "rerun">; - ad_hoc: boolean; - template_used: boolean; + runSource: RawDimension<"run" | "rerun">; + adHoc: boolean; + templateUsed: boolean; }, ): void { client.track( // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): measure Skill Studio validation loop usage "skill.test_run", { - skill_id: dims.skill_id, - status: dims.status, - run_source: dims.run_source, - ad_hoc: dims.ad_hoc, - template_used: dims.template_used, + skill_id: dims.skillId, + status: asEventDimension(dims.status), + run_source: asEventDimension(dims.runSource), + ad_hoc: dims.adHoc, + template_used: dims.templateUsed, }, ); } @@ -160,22 +160,22 @@ export function trackSkillTestRun( export function trackSkillForked( client: TelemetryClient, dims: { - skill_id: string; - fork_from_skill_id: string; - source_type: RawDimension<"local_path" | "github" | "url" | "catalog" | "skills_sh">; - sharing_scope: RawDimension<"private" | "company" | "public_link">; - reassign_agent_count: number; + skillId: string; + forkFromSkillId: string; + sourceType: RawDimension<"local_path" | "github" | "url" | "catalog" | "skills_sh">; + sharingScope: RawDimension<"private" | "company" | "public_link">; + reassignAgentCount: number; }, ): void { client.track( // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): measure Skill Studio fork completion and reassignment demand "skill.forked", { - skill_id: dims.skill_id, - fork_from_skill_id: dims.fork_from_skill_id, - source_type: dims.source_type, - sharing_scope: dims.sharing_scope, - reassign_agent_count: dims.reassign_agent_count, + skill_id: dims.skillId, + fork_from_skill_id: dims.forkFromSkillId, + source_type: asEventDimension(dims.sourceType), + sharing_scope: asEventDimension(dims.sharingScope), + reassign_agent_count: dims.reassignAgentCount, }, ); } @@ -183,14 +183,14 @@ export function trackSkillForked( export function trackSkillShareLinkCopied( client: TelemetryClient, dims: { - sharing_scope: RawDimension<"private" | "company" | "public_link">; + sharingScope: RawDimension<"private" | "company" | "public_link">; }, ): void { client.track( // @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): exercise triage of low-signal share-link copy proposal "skill.share_link", { - sharing_scope: dims.sharing_scope, + sharing_scope: asEventDimension(dims.sharingScope), }, ); } diff --git a/scripts/extract-proposed-events.mjs b/scripts/extract-proposed-events.mjs index ce29ace3fc8..fb9e3643030 100644 --- a/scripts/extract-proposed-events.mjs +++ b/scripts/extract-proposed-events.mjs @@ -75,11 +75,11 @@ export function extractProposedEvents(options = {}) { const sourceFile = ts.createSourceFile(eventsFile, sourceText, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS); const proposals = new Map(); - function recordProposal(wrapper, wrapperName, eventNameNode, directive) { + function recordProposal(wrapper, wrapperName, trackCall, eventNameNode, directive) { const name = eventNameNode.text; assertEventName(name, "event name"); const position = sourceFile.getLineAndCharacterOfPosition(eventNameNode.getStart(sourceFile)); - const dimensions = extractWrapperDimensions(wrapper, sourceFile, wrapperName); + const dimensions = extractWrapperDimensions(wrapper, sourceFile, wrapperName, trackCall.arguments[1]); let proposal = proposals.get(name); if (!proposal) { @@ -118,7 +118,7 @@ export function extractProposedEvents(options = {}) { const eventNameNode = node.arguments[0]; if (eventNameNode && ts.isStringLiteral(eventNameNode)) { const directive = findDirectiveForNode(sourceText, eventNameNode); - if (directive) recordProposal(wrapper, wrapperName, eventNameNode, directive); + if (directive) recordProposal(wrapper, wrapperName, node, eventNameNode, directive); } } ts.forEachChild(node, visit); @@ -193,11 +193,26 @@ function normalizeComment(commentText) { .trim(); } -function extractWrapperDimensions(wrapper, sourceFile, wrapperName = wrapper.name?.text ?? "") { +function extractWrapperDimensions(wrapper, sourceFile, wrapperName = wrapper.name?.text ?? "", payloadNode) { const dimsParam = wrapper.parameters.find( (parameter) => ts.isIdentifier(parameter.name) && parameter.name.text === "dims", ); - if (!dimsParam) return []; + + if (!payloadNode) return []; + + if (ts.isObjectLiteralExpression(payloadNode)) { + const dimsMembers = dimsParam ? extractDimsMembers(dimsParam, wrapperName) : new Map(); + return extractPayloadDimensions(payloadNode, dimsMembers, sourceFile); + } + + if (!dimsParam || !ts.isIdentifier(payloadNode) || payloadNode.text !== "dims") { + throw new Error(`wrapper ${wrapperName} proposed telemetry payload must be a dims identifier or object literal`); + } + + return [...extractDimsMembers(dimsParam, wrapperName).values()].map((member) => extractDimension(member, sourceFile)); +} + +function extractDimsMembers(dimsParam, wrapperName) { if (!dimsParam.type) { throw new Error(`wrapper ${wrapperName} has an untyped dims parameter`); } @@ -206,7 +221,77 @@ function extractWrapperDimensions(wrapper, sourceFile, wrapperName = wrapper.nam throw new Error(`wrapper ${wrapperName} dims parameter must be a type literal`); } - return typeNode.members.map((member) => extractDimension(member, sourceFile)); + const members = new Map(); + for (const member of typeNode.members) { + if (!ts.isPropertySignature(member) || !member.type) { + throw new Error("dims parameter type may contain only typed property signatures"); + } + members.set(propertyNameText(member.name), member); + } + return members; +} + +function extractPayloadDimensions(payloadNode, dimsMembers, sourceFile) { + const dimensions = []; + for (const property of payloadNode.properties) { + if (ts.isSpreadAssignment(property)) { + if (ts.isIdentifier(property.expression) && property.expression.text === "dims") { + for (const member of dimsMembers.values()) { + dimensions.push(extractDimension(member, sourceFile)); + } + continue; + } + throw new Error(`unsupported proposed telemetry spread payload: ${property.getText(sourceFile)}`); + } + + if (ts.isShorthandPropertyAssignment(property)) { + const name = propertyNameText(property.name); + assertEventName(name, "dimension name"); + const member = dimsMembers.get(name); + if (!member) throw new Error(`unable to infer proposed telemetry dimension type for ${name}`); + dimensions.push({ name, type: classifyTypeNode(member.type, sourceFile) }); + continue; + } + + if (!ts.isPropertyAssignment(property)) { + throw new Error(`unsupported proposed telemetry payload property: ${property.getText(sourceFile)}`); + } + + const name = propertyNameText(property.name); + assertEventName(name, "dimension name"); + dimensions.push({ name, type: classifyPayloadExpression(property.initializer, dimsMembers, sourceFile) }); + } + return dimensions; +} + +function classifyPayloadExpression(expression, dimsMembers, sourceFile) { + const dimsProperty = unwrapEventDimensionExpression(expression); + if (dimsProperty) { + const member = dimsMembers.get(dimsProperty); + if (!member) throw new Error(`unable to infer proposed telemetry dimension type for dims.${dimsProperty}`); + return classifyTypeNode(member.type, sourceFile); + } + + if (ts.isStringLiteral(expression) || expression.kind === ts.SyntaxKind.NoSubstitutionTemplateLiteral) return "string"; + if (ts.isNumericLiteral(expression)) return "number"; + if (expression.kind === ts.SyntaxKind.TrueKeyword || expression.kind === ts.SyntaxKind.FalseKeyword) return "boolean"; + + throw new Error(`unsupported proposed telemetry payload value: ${expression.getText(sourceFile)}`); +} + +function unwrapEventDimensionExpression(expression) { + if (ts.isPropertyAccessExpression(expression) && ts.isIdentifier(expression.expression) && expression.expression.text === "dims") { + return expression.name.text; + } + + if (ts.isCallExpression(expression)) { + const callee = expression.expression; + if (ts.isIdentifier(callee) && callee.text === "asEventDimension" && expression.arguments.length === 1) { + return unwrapEventDimensionExpression(expression.arguments[0]); + } + } + + return null; } function extractDimension(member, sourceFile) { diff --git a/scripts/extract-proposed-events.test.mjs b/scripts/extract-proposed-events.test.mjs index 6f3181ff9ce..7b473dc8a71 100644 --- a/scripts/extract-proposed-events.test.mjs +++ b/scripts/extract-proposed-events.test.mjs @@ -96,6 +96,44 @@ test("extractor emits deterministic proposed-telemetry-extractor.v2 records", () assert.equal(typeof created.provenance[0].column, "number"); }); +test("extractor infers proposed dimensions from snake_case payload mapped from camelCase wrapper dims", () => { + const output = withFixtureRepo( + `type TelemetryClient = { track(name: string, dims: unknown): void }; +type RawDimension = T | (string & {}); +function asEventDimension(value: T): T { return value; } + +export function trackSkillCreated( + client: TelemetryClient, + dims: { + skillId: string; + sharingScope: RawDimension<"private" | "company">; + categoryCount: number; + adHoc: boolean; + }, +): void { + client.track( + // @ts-expect-error -- proposed-telemetry(PAP-2411): measure skill creation + "skill.created", + { + skill_id: dims.skillId, + sharing_scope: asEventDimension(dims.sharingScope), + category_count: dims.categoryCount, + ad_hoc: dims.adHoc, + }, + ); +} +`, + ({ repoRoot, eventsFile }) => extractProposedEvents({ repoRoot, eventsFile, ref: "fixture-sha" }), + ); + + assert.deepEqual(output.proposals[0].dimensions, [ + { name: "ad_hoc", type: "boolean" }, + { name: "category_count", type: "number" }, + { name: "sharing_scope", type: "string" }, + { name: "skill_id", type: "string" }, + ]); +}); + test("extractor flags a missing proposed-telemetry suffix without hard-failing", () => { const output = withFixtureRepo(fixtureSource, ({ repoRoot, eventsFile }) => extractProposedEvents({ repoRoot, eventsFile, ref: "fixture-sha" }), diff --git a/server/src/__tests__/shared-telemetry-events.test.ts b/server/src/__tests__/shared-telemetry-events.test.ts index a6959bf9317..c2a190e4eb7 100644 --- a/server/src/__tests__/shared-telemetry-events.test.ts +++ b/server/src/__tests__/shared-telemetry-events.test.ts @@ -212,11 +212,11 @@ describe("shared Skill telemetry proposals", () => { it("emits primitive skill.created dimensions and omits user-authored identity", () => { const client = createClient(); const dims = { - skill_id: "11111111-1111-4111-8111-111111111111", - creation_source: "blank", - sharing_scope: "company", - category_count: 2, - file_count: 3, + skillId: "11111111-1111-4111-8111-111111111111", + creationSource: "blank", + sharingScope: "company", + categoryCount: 2, + fileCount: 3, name: "Secret Skill", slug: "secret-skill", tagline: "keep this private", @@ -242,9 +242,9 @@ describe("shared Skill telemetry proposals", () => { it("emits primitive skill.version_saved dimensions without file path or content", () => { const client = createClient(); const dims = { - skill_id: "22222222-2222-4222-8222-222222222222", - revision_number: 4, - file_type: "markdown", + skillId: "22222222-2222-4222-8222-222222222222", + revisionNumber: 4, + fileType: "markdown", file_path: "customers/acme/SKILL.md", file_content: "keep this private", path: "customers/acme/SKILL.md", @@ -269,11 +269,11 @@ describe("shared Skill telemetry proposals", () => { it("emits primitive skill.test_run dimensions without snapshots, prompts, or errors", () => { const client = createClient(); const dims = { - skill_id: "33333333-3333-4333-8333-333333333333", + skillId: "33333333-3333-4333-8333-333333333333", status: "queued", - run_source: "run", - ad_hoc: true, - template_used: false, + runSource: "run", + adHoc: true, + templateUsed: false, inputSnapshot: "do not leak input", outputSnapshot: "do not leak output", harnessIssueDescription: "private harness issue", @@ -302,11 +302,11 @@ describe("shared Skill telemetry proposals", () => { it("emits primitive skill.forked dimensions and passes opaque IDs through raw", () => { const client = createClient(); const dims = { - skill_id: "44444444-4444-4444-8444-444444444444", - fork_from_skill_id: "55555555-5555-4555-8555-555555555555", - source_type: "github", - sharing_scope: "private", - reassign_agent_count: 1, + skillId: "44444444-4444-4444-8444-444444444444", + forkFromSkillId: "55555555-5555-4555-8555-555555555555", + sourceType: "github", + sharingScope: "private", + reassignAgentCount: 1, name: "Secret Skill", slug: "secret-skill", fork_from_name: "Secret Source", @@ -332,7 +332,7 @@ describe("shared Skill telemetry proposals", () => { it("emits only sharing_scope for the deliberately rejectable share-link proposal", () => { const client = createClient(); const dims = { - sharing_scope: "public_link", + sharingScope: "public_link", skill_id: "66666666-6666-4666-8666-666666666666", href: "https://paperclip.example/skills/secret-skill?token=secret-token", url: "https://paperclip.example/skills/secret-skill?token=secret-token",