fix: align proposed telemetry dims

Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Nicky LeachandPaperclip committed 2026-07-14 16:39:13 +00:00
1 parent 2f3a61a116
commit 2c3d3fb652
5 files changed
+202 -79

No files matched your search

+18 -18
View File
@@ -58,32 +58,32 @@ describe("TelemetryClient runtime event gate", () => {
const { client, stateFactory } = makeClient();
trackSkillCreated(client, {
skill_id: "11111111-1111-4111-8111-111111111111",
creation_source: "blank",
sharing_scope: "company",
category_count: 1,
file_count: 2,
skillId: "11111111-1111-4111-8111-111111111111",
creationSource: "blank",
sharingScope: "company",
categoryCount: 1,
fileCount: 2,
});
trackSkillVersionSaved(client, {
skill_id: "11111111-1111-4111-8111-111111111111",
revision_number: 2,
file_type: "skill",
skillId: "11111111-1111-4111-8111-111111111111",
revisionNumber: 2,
fileType: "skill",
});
trackSkillTestRun(client, {
skill_id: "11111111-1111-4111-8111-111111111111",
skillId: "11111111-1111-4111-8111-111111111111",
status: "queued",
run_source: "run",
ad_hoc: true,
template_used: false,
runSource: "run",
adHoc: true,
templateUsed: false,
});
trackSkillForked(client, {
skill_id: "22222222-2222-4222-8222-222222222222",
fork_from_skill_id: "11111111-1111-4111-8111-111111111111",
source_type: "catalog",
sharing_scope: "private",
reassign_agent_count: 0,
skillId: "22222222-2222-4222-8222-222222222222",
forkFromSkillId: "11111111-1111-4111-8111-111111111111",
sourceType: "catalog",
sharingScope: "private",
reassignAgentCount: 0,
});
trackSkillShareLinkCopied(client, { sharing_scope: "public_link" });
trackSkillShareLinkCopied(client, { sharingScope: "public_link" });
await client.flush();
+37 -37
View File
@@ -95,22 +95,22 @@ export function trackSkillImported(
export function trackSkillCreated(
client: TelemetryClient,
dims: {
skill_id: string;
creation_source: RawDimension<"blank" | "fork" | "project_scan">;
sharing_scope: RawDimension<"private" | "company" | "public_link">;
category_count: number;
file_count: number;
skillId: string;
creationSource: RawDimension<"blank" | "fork" | "project_scan">;
sharingScope: RawDimension<"private" | "company" | "public_link">;
categoryCount: number;
fileCount: number;
},
): void {
client.track(
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): measure successful Skill Studio skill creation funnel
"skill.created",
{
skill_id: dims.skill_id,
creation_source: dims.creation_source,
sharing_scope: dims.sharing_scope,
category_count: dims.category_count,
file_count: dims.file_count,
skill_id: dims.skillId,
creation_source: asEventDimension(dims.creationSource),
sharing_scope: asEventDimension(dims.sharingScope),
category_count: dims.categoryCount,
file_count: dims.fileCount,
},
);
}
@@ -118,18 +118,18 @@ export function trackSkillCreated(
export function trackSkillVersionSaved(
client: TelemetryClient,
dims: {
skill_id: string;
revision_number: number;
file_type: RawDimension<"skill" | "markdown" | "reference" | "script" | "asset" | "other">;
skillId: string;
revisionNumber: number;
fileType: RawDimension<"skill" | "markdown" | "reference" | "script" | "asset" | "other">;
},
): void {
client.track(
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): measure Skill Studio editor save and version creation usage
"skill.version_saved",
{
skill_id: dims.skill_id,
revision_number: dims.revision_number,
file_type: dims.file_type,
skill_id: dims.skillId,
revision_number: dims.revisionNumber,
file_type: asEventDimension(dims.fileType),
},
);
}
@@ -137,22 +137,22 @@ export function trackSkillVersionSaved(
export function trackSkillTestRun(
client: TelemetryClient,
dims: {
skill_id: string;
skillId: string;
status: RawDimension<"queued" | "running" | "succeeded" | "failed" | "cancelled">;
run_source: RawDimension<"run" | "rerun">;
ad_hoc: boolean;
template_used: boolean;
runSource: RawDimension<"run" | "rerun">;
adHoc: boolean;
templateUsed: boolean;
},
): void {
client.track(
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): measure Skill Studio validation loop usage
"skill.test_run",
{
skill_id: dims.skill_id,
status: dims.status,
run_source: dims.run_source,
ad_hoc: dims.ad_hoc,
template_used: dims.template_used,
skill_id: dims.skillId,
status: asEventDimension(dims.status),
run_source: asEventDimension(dims.runSource),
ad_hoc: dims.adHoc,
template_used: dims.templateUsed,
},
);
}
@@ -160,22 +160,22 @@ export function trackSkillTestRun(
export function trackSkillForked(
client: TelemetryClient,
dims: {
skill_id: string;
fork_from_skill_id: string;
source_type: RawDimension<"local_path" | "github" | "url" | "catalog" | "skills_sh">;
sharing_scope: RawDimension<"private" | "company" | "public_link">;
reassign_agent_count: number;
skillId: string;
forkFromSkillId: string;
sourceType: RawDimension<"local_path" | "github" | "url" | "catalog" | "skills_sh">;
sharingScope: RawDimension<"private" | "company" | "public_link">;
reassignAgentCount: number;
},
): void {
client.track(
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): measure Skill Studio fork completion and reassignment demand
"skill.forked",
{
skill_id: dims.skill_id,
fork_from_skill_id: dims.fork_from_skill_id,
source_type: dims.source_type,
sharing_scope: dims.sharing_scope,
reassign_agent_count: dims.reassign_agent_count,
skill_id: dims.skillId,
fork_from_skill_id: dims.forkFromSkillId,
source_type: asEventDimension(dims.sourceType),
sharing_scope: asEventDimension(dims.sharingScope),
reassign_agent_count: dims.reassignAgentCount,
},
);
}
@@ -183,14 +183,14 @@ export function trackSkillForked(
export function trackSkillShareLinkCopied(
client: TelemetryClient,
dims: {
sharing_scope: RawDimension<"private" | "company" | "public_link">;
sharingScope: RawDimension<"private" | "company" | "public_link">;
},
): void {
client.track(
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): exercise triage of low-signal share-link copy proposal
"skill.share_link",
{
sharing_scope: dims.sharing_scope,
sharing_scope: asEventDimension(dims.sharingScope),
},
);
}
+91 -6
View File
@@ -75,11 +75,11 @@ export function extractProposedEvents(options = {}) {
const sourceFile = ts.createSourceFile(eventsFile, sourceText, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);
const proposals = new Map();
function recordProposal(wrapper, wrapperName, eventNameNode, directive) {
function recordProposal(wrapper, wrapperName, trackCall, eventNameNode, directive) {
const name = eventNameNode.text;
assertEventName(name, "event name");
const position = sourceFile.getLineAndCharacterOfPosition(eventNameNode.getStart(sourceFile));
const dimensions = extractWrapperDimensions(wrapper, sourceFile, wrapperName);
const dimensions = extractWrapperDimensions(wrapper, sourceFile, wrapperName, trackCall.arguments[1]);
let proposal = proposals.get(name);
if (!proposal) {
@@ -118,7 +118,7 @@ export function extractProposedEvents(options = {}) {
const eventNameNode = node.arguments[0];
if (eventNameNode && ts.isStringLiteral(eventNameNode)) {
const directive = findDirectiveForNode(sourceText, eventNameNode);
if (directive) recordProposal(wrapper, wrapperName, eventNameNode, directive);
if (directive) recordProposal(wrapper, wrapperName, node, eventNameNode, directive);
}
}
ts.forEachChild(node, visit);
@@ -193,11 +193,26 @@ function normalizeComment(commentText) {
.trim();
}
function extractWrapperDimensions(wrapper, sourceFile, wrapperName = wrapper.name?.text ?? "<anonymous>") {
function extractWrapperDimensions(wrapper, sourceFile, wrapperName = wrapper.name?.text ?? "<anonymous>", payloadNode) {
const dimsParam = wrapper.parameters.find(
(parameter) => ts.isIdentifier(parameter.name) && parameter.name.text === "dims",
);
if (!dimsParam) return [];
if (!payloadNode) return [];
if (ts.isObjectLiteralExpression(payloadNode)) {
const dimsMembers = dimsParam ? extractDimsMembers(dimsParam, wrapperName) : new Map();
return extractPayloadDimensions(payloadNode, dimsMembers, sourceFile);
}
if (!dimsParam || !ts.isIdentifier(payloadNode) || payloadNode.text !== "dims") {
throw new Error(`wrapper ${wrapperName} proposed telemetry payload must be a dims identifier or object literal`);
}
return [...extractDimsMembers(dimsParam, wrapperName).values()].map((member) => extractDimension(member, sourceFile));
}
function extractDimsMembers(dimsParam, wrapperName) {
if (!dimsParam.type) {
throw new Error(`wrapper ${wrapperName} has an untyped dims parameter`);
}
@@ -206,7 +221,77 @@ function extractWrapperDimensions(wrapper, sourceFile, wrapperName = wrapper.nam
throw new Error(`wrapper ${wrapperName} dims parameter must be a type literal`);
}
return typeNode.members.map((member) => extractDimension(member, sourceFile));
const members = new Map();
for (const member of typeNode.members) {
if (!ts.isPropertySignature(member) || !member.type) {
throw new Error("dims parameter type may contain only typed property signatures");
}
members.set(propertyNameText(member.name), member);
}
return members;
}
function extractPayloadDimensions(payloadNode, dimsMembers, sourceFile) {
const dimensions = [];
for (const property of payloadNode.properties) {
if (ts.isSpreadAssignment(property)) {
if (ts.isIdentifier(property.expression) && property.expression.text === "dims") {
for (const member of dimsMembers.values()) {
dimensions.push(extractDimension(member, sourceFile));
}
continue;
}
throw new Error(`unsupported proposed telemetry spread payload: ${property.getText(sourceFile)}`);
}
if (ts.isShorthandPropertyAssignment(property)) {
const name = propertyNameText(property.name);
assertEventName(name, "dimension name");
const member = dimsMembers.get(name);
if (!member) throw new Error(`unable to infer proposed telemetry dimension type for ${name}`);
dimensions.push({ name, type: classifyTypeNode(member.type, sourceFile) });
continue;
}
if (!ts.isPropertyAssignment(property)) {
throw new Error(`unsupported proposed telemetry payload property: ${property.getText(sourceFile)}`);
}
const name = propertyNameText(property.name);
assertEventName(name, "dimension name");
dimensions.push({ name, type: classifyPayloadExpression(property.initializer, dimsMembers, sourceFile) });
}
return dimensions;
}
function classifyPayloadExpression(expression, dimsMembers, sourceFile) {
const dimsProperty = unwrapEventDimensionExpression(expression);
if (dimsProperty) {
const member = dimsMembers.get(dimsProperty);
if (!member) throw new Error(`unable to infer proposed telemetry dimension type for dims.${dimsProperty}`);
return classifyTypeNode(member.type, sourceFile);
}
if (ts.isStringLiteral(expression) || expression.kind === ts.SyntaxKind.NoSubstitutionTemplateLiteral) return "string";
if (ts.isNumericLiteral(expression)) return "number";
if (expression.kind === ts.SyntaxKind.TrueKeyword || expression.kind === ts.SyntaxKind.FalseKeyword) return "boolean";
throw new Error(`unsupported proposed telemetry payload value: ${expression.getText(sourceFile)}`);
}
function unwrapEventDimensionExpression(expression) {
if (ts.isPropertyAccessExpression(expression) && ts.isIdentifier(expression.expression) && expression.expression.text === "dims") {
return expression.name.text;
}
if (ts.isCallExpression(expression)) {
const callee = expression.expression;
if (ts.isIdentifier(callee) && callee.text === "asEventDimension" && expression.arguments.length === 1) {
return unwrapEventDimensionExpression(expression.arguments[0]);
}
}
return null;
}
function extractDimension(member, sourceFile) {
+38
View File
@@ -96,6 +96,44 @@ test("extractor emits deterministic proposed-telemetry-extractor.v2 records", ()
assert.equal(typeof created.provenance[0].column, "number");
});
test("extractor infers proposed dimensions from snake_case payload mapped from camelCase wrapper dims", () => {
const output = withFixtureRepo(
`type TelemetryClient = { track(name: string, dims: unknown): void };
type RawDimension<T extends string | undefined> = T | (string & {});
function asEventDimension<T extends string>(value: T): T { return value; }
export function trackSkillCreated(
client: TelemetryClient,
dims: {
skillId: string;
sharingScope: RawDimension<"private" | "company">;
categoryCount: number;
adHoc: boolean;
},
): void {
client.track(
// @ts-expect-error -- proposed-telemetry(PAP-2411): measure skill creation
"skill.created",
{
skill_id: dims.skillId,
sharing_scope: asEventDimension(dims.sharingScope),
category_count: dims.categoryCount,
ad_hoc: dims.adHoc,
},
);
}
`,
({ repoRoot, eventsFile }) => extractProposedEvents({ repoRoot, eventsFile, ref: "fixture-sha" }),
);
assert.deepEqual(output.proposals[0].dimensions, [
{ name: "ad_hoc", type: "boolean" },
{ name: "category_count", type: "number" },
{ name: "sharing_scope", type: "string" },
{ name: "skill_id", type: "string" },
]);
});
test("extractor flags a missing proposed-telemetry suffix without hard-failing", () => {
const output = withFixtureRepo(fixtureSource, ({ repoRoot, eventsFile }) =>
extractProposedEvents({ repoRoot, eventsFile, ref: "fixture-sha" }),
@@ -212,11 +212,11 @@ describe("shared Skill telemetry proposals", () => {
it("emits primitive skill.created dimensions and omits user-authored identity", () => {
const client = createClient();
const dims = {
skill_id: "11111111-1111-4111-8111-111111111111",
creation_source: "blank",
sharing_scope: "company",
category_count: 2,
file_count: 3,
skillId: "11111111-1111-4111-8111-111111111111",
creationSource: "blank",
sharingScope: "company",
categoryCount: 2,
fileCount: 3,
name: "Secret Skill",
slug: "secret-skill",
tagline: "keep this private",
@@ -242,9 +242,9 @@ describe("shared Skill telemetry proposals", () => {
it("emits primitive skill.version_saved dimensions without file path or content", () => {
const client = createClient();
const dims = {
skill_id: "22222222-2222-4222-8222-222222222222",
revision_number: 4,
file_type: "markdown",
skillId: "22222222-2222-4222-8222-222222222222",
revisionNumber: 4,
fileType: "markdown",
file_path: "customers/acme/SKILL.md",
file_content: "keep this private",
path: "customers/acme/SKILL.md",
@@ -269,11 +269,11 @@ describe("shared Skill telemetry proposals", () => {
it("emits primitive skill.test_run dimensions without snapshots, prompts, or errors", () => {
const client = createClient();
const dims = {
skill_id: "33333333-3333-4333-8333-333333333333",
skillId: "33333333-3333-4333-8333-333333333333",
status: "queued",
run_source: "run",
ad_hoc: true,
template_used: false,
runSource: "run",
adHoc: true,
templateUsed: false,
inputSnapshot: "do not leak input",
outputSnapshot: "do not leak output",
harnessIssueDescription: "private harness issue",
@@ -302,11 +302,11 @@ describe("shared Skill telemetry proposals", () => {
it("emits primitive skill.forked dimensions and passes opaque IDs through raw", () => {
const client = createClient();
const dims = {
skill_id: "44444444-4444-4444-8444-444444444444",
fork_from_skill_id: "55555555-5555-4555-8555-555555555555",
source_type: "github",
sharing_scope: "private",
reassign_agent_count: 1,
skillId: "44444444-4444-4444-8444-444444444444",
forkFromSkillId: "55555555-5555-4555-8555-555555555555",
sourceType: "github",
sharingScope: "private",
reassignAgentCount: 1,
name: "Secret Skill",
slug: "secret-skill",
fork_from_name: "Secret Source",
@@ -332,7 +332,7 @@ describe("shared Skill telemetry proposals", () => {
it("emits only sharing_scope for the deliberately rejectable share-link proposal", () => {
const client = createClient();
const dims = {
sharing_scope: "public_link",
sharingScope: "public_link",
skill_id: "66666666-6666-4666-8666-666666666666",
href: "https://paperclip.example/skills/secret-skill?token=secret-token",
url: "https://paperclip.example/skills/secret-skill?token=secret-token",