mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-26 02:00:52 +01:00
fix: align proposed telemetry dims
Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
5 files changed
+202
-79
No files matched your search
@@ -58,32 +58,32 @@ describe("TelemetryClient runtime event gate", () => {
|
||||
const { client, stateFactory } = makeClient();
|
||||
|
||||
trackSkillCreated(client, {
|
||||
skill_id: "11111111-1111-4111-8111-111111111111",
|
||||
creation_source: "blank",
|
||||
sharing_scope: "company",
|
||||
category_count: 1,
|
||||
file_count: 2,
|
||||
skillId: "11111111-1111-4111-8111-111111111111",
|
||||
creationSource: "blank",
|
||||
sharingScope: "company",
|
||||
categoryCount: 1,
|
||||
fileCount: 2,
|
||||
});
|
||||
trackSkillVersionSaved(client, {
|
||||
skill_id: "11111111-1111-4111-8111-111111111111",
|
||||
revision_number: 2,
|
||||
file_type: "skill",
|
||||
skillId: "11111111-1111-4111-8111-111111111111",
|
||||
revisionNumber: 2,
|
||||
fileType: "skill",
|
||||
});
|
||||
trackSkillTestRun(client, {
|
||||
skill_id: "11111111-1111-4111-8111-111111111111",
|
||||
skillId: "11111111-1111-4111-8111-111111111111",
|
||||
status: "queued",
|
||||
run_source: "run",
|
||||
ad_hoc: true,
|
||||
template_used: false,
|
||||
runSource: "run",
|
||||
adHoc: true,
|
||||
templateUsed: false,
|
||||
});
|
||||
trackSkillForked(client, {
|
||||
skill_id: "22222222-2222-4222-8222-222222222222",
|
||||
fork_from_skill_id: "11111111-1111-4111-8111-111111111111",
|
||||
source_type: "catalog",
|
||||
sharing_scope: "private",
|
||||
reassign_agent_count: 0,
|
||||
skillId: "22222222-2222-4222-8222-222222222222",
|
||||
forkFromSkillId: "11111111-1111-4111-8111-111111111111",
|
||||
sourceType: "catalog",
|
||||
sharingScope: "private",
|
||||
reassignAgentCount: 0,
|
||||
});
|
||||
trackSkillShareLinkCopied(client, { sharing_scope: "public_link" });
|
||||
trackSkillShareLinkCopied(client, { sharingScope: "public_link" });
|
||||
|
||||
await client.flush();
|
||||
|
||||
|
||||
@@ -95,22 +95,22 @@ export function trackSkillImported(
|
||||
export function trackSkillCreated(
|
||||
client: TelemetryClient,
|
||||
dims: {
|
||||
skill_id: string;
|
||||
creation_source: RawDimension<"blank" | "fork" | "project_scan">;
|
||||
sharing_scope: RawDimension<"private" | "company" | "public_link">;
|
||||
category_count: number;
|
||||
file_count: number;
|
||||
skillId: string;
|
||||
creationSource: RawDimension<"blank" | "fork" | "project_scan">;
|
||||
sharingScope: RawDimension<"private" | "company" | "public_link">;
|
||||
categoryCount: number;
|
||||
fileCount: number;
|
||||
},
|
||||
): void {
|
||||
client.track(
|
||||
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): measure successful Skill Studio skill creation funnel
|
||||
"skill.created",
|
||||
{
|
||||
skill_id: dims.skill_id,
|
||||
creation_source: dims.creation_source,
|
||||
sharing_scope: dims.sharing_scope,
|
||||
category_count: dims.category_count,
|
||||
file_count: dims.file_count,
|
||||
skill_id: dims.skillId,
|
||||
creation_source: asEventDimension(dims.creationSource),
|
||||
sharing_scope: asEventDimension(dims.sharingScope),
|
||||
category_count: dims.categoryCount,
|
||||
file_count: dims.fileCount,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -118,18 +118,18 @@ export function trackSkillCreated(
|
||||
export function trackSkillVersionSaved(
|
||||
client: TelemetryClient,
|
||||
dims: {
|
||||
skill_id: string;
|
||||
revision_number: number;
|
||||
file_type: RawDimension<"skill" | "markdown" | "reference" | "script" | "asset" | "other">;
|
||||
skillId: string;
|
||||
revisionNumber: number;
|
||||
fileType: RawDimension<"skill" | "markdown" | "reference" | "script" | "asset" | "other">;
|
||||
},
|
||||
): void {
|
||||
client.track(
|
||||
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): measure Skill Studio editor save and version creation usage
|
||||
"skill.version_saved",
|
||||
{
|
||||
skill_id: dims.skill_id,
|
||||
revision_number: dims.revision_number,
|
||||
file_type: dims.file_type,
|
||||
skill_id: dims.skillId,
|
||||
revision_number: dims.revisionNumber,
|
||||
file_type: asEventDimension(dims.fileType),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -137,22 +137,22 @@ export function trackSkillVersionSaved(
|
||||
export function trackSkillTestRun(
|
||||
client: TelemetryClient,
|
||||
dims: {
|
||||
skill_id: string;
|
||||
skillId: string;
|
||||
status: RawDimension<"queued" | "running" | "succeeded" | "failed" | "cancelled">;
|
||||
run_source: RawDimension<"run" | "rerun">;
|
||||
ad_hoc: boolean;
|
||||
template_used: boolean;
|
||||
runSource: RawDimension<"run" | "rerun">;
|
||||
adHoc: boolean;
|
||||
templateUsed: boolean;
|
||||
},
|
||||
): void {
|
||||
client.track(
|
||||
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): measure Skill Studio validation loop usage
|
||||
"skill.test_run",
|
||||
{
|
||||
skill_id: dims.skill_id,
|
||||
status: dims.status,
|
||||
run_source: dims.run_source,
|
||||
ad_hoc: dims.ad_hoc,
|
||||
template_used: dims.template_used,
|
||||
skill_id: dims.skillId,
|
||||
status: asEventDimension(dims.status),
|
||||
run_source: asEventDimension(dims.runSource),
|
||||
ad_hoc: dims.adHoc,
|
||||
template_used: dims.templateUsed,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -160,22 +160,22 @@ export function trackSkillTestRun(
|
||||
export function trackSkillForked(
|
||||
client: TelemetryClient,
|
||||
dims: {
|
||||
skill_id: string;
|
||||
fork_from_skill_id: string;
|
||||
source_type: RawDimension<"local_path" | "github" | "url" | "catalog" | "skills_sh">;
|
||||
sharing_scope: RawDimension<"private" | "company" | "public_link">;
|
||||
reassign_agent_count: number;
|
||||
skillId: string;
|
||||
forkFromSkillId: string;
|
||||
sourceType: RawDimension<"local_path" | "github" | "url" | "catalog" | "skills_sh">;
|
||||
sharingScope: RawDimension<"private" | "company" | "public_link">;
|
||||
reassignAgentCount: number;
|
||||
},
|
||||
): void {
|
||||
client.track(
|
||||
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): measure Skill Studio fork completion and reassignment demand
|
||||
"skill.forked",
|
||||
{
|
||||
skill_id: dims.skill_id,
|
||||
fork_from_skill_id: dims.fork_from_skill_id,
|
||||
source_type: dims.source_type,
|
||||
sharing_scope: dims.sharing_scope,
|
||||
reassign_agent_count: dims.reassign_agent_count,
|
||||
skill_id: dims.skillId,
|
||||
fork_from_skill_id: dims.forkFromSkillId,
|
||||
source_type: asEventDimension(dims.sourceType),
|
||||
sharing_scope: asEventDimension(dims.sharingScope),
|
||||
reassign_agent_count: dims.reassignAgentCount,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -183,14 +183,14 @@ export function trackSkillForked(
|
||||
export function trackSkillShareLinkCopied(
|
||||
client: TelemetryClient,
|
||||
dims: {
|
||||
sharing_scope: RawDimension<"private" | "company" | "public_link">;
|
||||
sharingScope: RawDimension<"private" | "company" | "public_link">;
|
||||
},
|
||||
): void {
|
||||
client.track(
|
||||
// @ts-expect-error -- proposed-telemetry(https://github.com/paperclipai/paperclip/issues/2411): exercise triage of low-signal share-link copy proposal
|
||||
"skill.share_link",
|
||||
{
|
||||
sharing_scope: dims.sharing_scope,
|
||||
sharing_scope: asEventDimension(dims.sharingScope),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -75,11 +75,11 @@ export function extractProposedEvents(options = {}) {
|
||||
const sourceFile = ts.createSourceFile(eventsFile, sourceText, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);
|
||||
const proposals = new Map();
|
||||
|
||||
function recordProposal(wrapper, wrapperName, eventNameNode, directive) {
|
||||
function recordProposal(wrapper, wrapperName, trackCall, eventNameNode, directive) {
|
||||
const name = eventNameNode.text;
|
||||
assertEventName(name, "event name");
|
||||
const position = sourceFile.getLineAndCharacterOfPosition(eventNameNode.getStart(sourceFile));
|
||||
const dimensions = extractWrapperDimensions(wrapper, sourceFile, wrapperName);
|
||||
const dimensions = extractWrapperDimensions(wrapper, sourceFile, wrapperName, trackCall.arguments[1]);
|
||||
|
||||
let proposal = proposals.get(name);
|
||||
if (!proposal) {
|
||||
@@ -118,7 +118,7 @@ export function extractProposedEvents(options = {}) {
|
||||
const eventNameNode = node.arguments[0];
|
||||
if (eventNameNode && ts.isStringLiteral(eventNameNode)) {
|
||||
const directive = findDirectiveForNode(sourceText, eventNameNode);
|
||||
if (directive) recordProposal(wrapper, wrapperName, eventNameNode, directive);
|
||||
if (directive) recordProposal(wrapper, wrapperName, node, eventNameNode, directive);
|
||||
}
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
@@ -193,11 +193,26 @@ function normalizeComment(commentText) {
|
||||
.trim();
|
||||
}
|
||||
|
||||
function extractWrapperDimensions(wrapper, sourceFile, wrapperName = wrapper.name?.text ?? "<anonymous>") {
|
||||
function extractWrapperDimensions(wrapper, sourceFile, wrapperName = wrapper.name?.text ?? "<anonymous>", payloadNode) {
|
||||
const dimsParam = wrapper.parameters.find(
|
||||
(parameter) => ts.isIdentifier(parameter.name) && parameter.name.text === "dims",
|
||||
);
|
||||
if (!dimsParam) return [];
|
||||
|
||||
if (!payloadNode) return [];
|
||||
|
||||
if (ts.isObjectLiteralExpression(payloadNode)) {
|
||||
const dimsMembers = dimsParam ? extractDimsMembers(dimsParam, wrapperName) : new Map();
|
||||
return extractPayloadDimensions(payloadNode, dimsMembers, sourceFile);
|
||||
}
|
||||
|
||||
if (!dimsParam || !ts.isIdentifier(payloadNode) || payloadNode.text !== "dims") {
|
||||
throw new Error(`wrapper ${wrapperName} proposed telemetry payload must be a dims identifier or object literal`);
|
||||
}
|
||||
|
||||
return [...extractDimsMembers(dimsParam, wrapperName).values()].map((member) => extractDimension(member, sourceFile));
|
||||
}
|
||||
|
||||
function extractDimsMembers(dimsParam, wrapperName) {
|
||||
if (!dimsParam.type) {
|
||||
throw new Error(`wrapper ${wrapperName} has an untyped dims parameter`);
|
||||
}
|
||||
@@ -206,7 +221,77 @@ function extractWrapperDimensions(wrapper, sourceFile, wrapperName = wrapper.nam
|
||||
throw new Error(`wrapper ${wrapperName} dims parameter must be a type literal`);
|
||||
}
|
||||
|
||||
return typeNode.members.map((member) => extractDimension(member, sourceFile));
|
||||
const members = new Map();
|
||||
for (const member of typeNode.members) {
|
||||
if (!ts.isPropertySignature(member) || !member.type) {
|
||||
throw new Error("dims parameter type may contain only typed property signatures");
|
||||
}
|
||||
members.set(propertyNameText(member.name), member);
|
||||
}
|
||||
return members;
|
||||
}
|
||||
|
||||
function extractPayloadDimensions(payloadNode, dimsMembers, sourceFile) {
|
||||
const dimensions = [];
|
||||
for (const property of payloadNode.properties) {
|
||||
if (ts.isSpreadAssignment(property)) {
|
||||
if (ts.isIdentifier(property.expression) && property.expression.text === "dims") {
|
||||
for (const member of dimsMembers.values()) {
|
||||
dimensions.push(extractDimension(member, sourceFile));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
throw new Error(`unsupported proposed telemetry spread payload: ${property.getText(sourceFile)}`);
|
||||
}
|
||||
|
||||
if (ts.isShorthandPropertyAssignment(property)) {
|
||||
const name = propertyNameText(property.name);
|
||||
assertEventName(name, "dimension name");
|
||||
const member = dimsMembers.get(name);
|
||||
if (!member) throw new Error(`unable to infer proposed telemetry dimension type for ${name}`);
|
||||
dimensions.push({ name, type: classifyTypeNode(member.type, sourceFile) });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!ts.isPropertyAssignment(property)) {
|
||||
throw new Error(`unsupported proposed telemetry payload property: ${property.getText(sourceFile)}`);
|
||||
}
|
||||
|
||||
const name = propertyNameText(property.name);
|
||||
assertEventName(name, "dimension name");
|
||||
dimensions.push({ name, type: classifyPayloadExpression(property.initializer, dimsMembers, sourceFile) });
|
||||
}
|
||||
return dimensions;
|
||||
}
|
||||
|
||||
function classifyPayloadExpression(expression, dimsMembers, sourceFile) {
|
||||
const dimsProperty = unwrapEventDimensionExpression(expression);
|
||||
if (dimsProperty) {
|
||||
const member = dimsMembers.get(dimsProperty);
|
||||
if (!member) throw new Error(`unable to infer proposed telemetry dimension type for dims.${dimsProperty}`);
|
||||
return classifyTypeNode(member.type, sourceFile);
|
||||
}
|
||||
|
||||
if (ts.isStringLiteral(expression) || expression.kind === ts.SyntaxKind.NoSubstitutionTemplateLiteral) return "string";
|
||||
if (ts.isNumericLiteral(expression)) return "number";
|
||||
if (expression.kind === ts.SyntaxKind.TrueKeyword || expression.kind === ts.SyntaxKind.FalseKeyword) return "boolean";
|
||||
|
||||
throw new Error(`unsupported proposed telemetry payload value: ${expression.getText(sourceFile)}`);
|
||||
}
|
||||
|
||||
function unwrapEventDimensionExpression(expression) {
|
||||
if (ts.isPropertyAccessExpression(expression) && ts.isIdentifier(expression.expression) && expression.expression.text === "dims") {
|
||||
return expression.name.text;
|
||||
}
|
||||
|
||||
if (ts.isCallExpression(expression)) {
|
||||
const callee = expression.expression;
|
||||
if (ts.isIdentifier(callee) && callee.text === "asEventDimension" && expression.arguments.length === 1) {
|
||||
return unwrapEventDimensionExpression(expression.arguments[0]);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractDimension(member, sourceFile) {
|
||||
|
||||
@@ -96,6 +96,44 @@ test("extractor emits deterministic proposed-telemetry-extractor.v2 records", ()
|
||||
assert.equal(typeof created.provenance[0].column, "number");
|
||||
});
|
||||
|
||||
test("extractor infers proposed dimensions from snake_case payload mapped from camelCase wrapper dims", () => {
|
||||
const output = withFixtureRepo(
|
||||
`type TelemetryClient = { track(name: string, dims: unknown): void };
|
||||
type RawDimension<T extends string | undefined> = T | (string & {});
|
||||
function asEventDimension<T extends string>(value: T): T { return value; }
|
||||
|
||||
export function trackSkillCreated(
|
||||
client: TelemetryClient,
|
||||
dims: {
|
||||
skillId: string;
|
||||
sharingScope: RawDimension<"private" | "company">;
|
||||
categoryCount: number;
|
||||
adHoc: boolean;
|
||||
},
|
||||
): void {
|
||||
client.track(
|
||||
// @ts-expect-error -- proposed-telemetry(PAP-2411): measure skill creation
|
||||
"skill.created",
|
||||
{
|
||||
skill_id: dims.skillId,
|
||||
sharing_scope: asEventDimension(dims.sharingScope),
|
||||
category_count: dims.categoryCount,
|
||||
ad_hoc: dims.adHoc,
|
||||
},
|
||||
);
|
||||
}
|
||||
`,
|
||||
({ repoRoot, eventsFile }) => extractProposedEvents({ repoRoot, eventsFile, ref: "fixture-sha" }),
|
||||
);
|
||||
|
||||
assert.deepEqual(output.proposals[0].dimensions, [
|
||||
{ name: "ad_hoc", type: "boolean" },
|
||||
{ name: "category_count", type: "number" },
|
||||
{ name: "sharing_scope", type: "string" },
|
||||
{ name: "skill_id", type: "string" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("extractor flags a missing proposed-telemetry suffix without hard-failing", () => {
|
||||
const output = withFixtureRepo(fixtureSource, ({ repoRoot, eventsFile }) =>
|
||||
extractProposedEvents({ repoRoot, eventsFile, ref: "fixture-sha" }),
|
||||
|
||||
@@ -212,11 +212,11 @@ describe("shared Skill telemetry proposals", () => {
|
||||
it("emits primitive skill.created dimensions and omits user-authored identity", () => {
|
||||
const client = createClient();
|
||||
const dims = {
|
||||
skill_id: "11111111-1111-4111-8111-111111111111",
|
||||
creation_source: "blank",
|
||||
sharing_scope: "company",
|
||||
category_count: 2,
|
||||
file_count: 3,
|
||||
skillId: "11111111-1111-4111-8111-111111111111",
|
||||
creationSource: "blank",
|
||||
sharingScope: "company",
|
||||
categoryCount: 2,
|
||||
fileCount: 3,
|
||||
name: "Secret Skill",
|
||||
slug: "secret-skill",
|
||||
tagline: "keep this private",
|
||||
@@ -242,9 +242,9 @@ describe("shared Skill telemetry proposals", () => {
|
||||
it("emits primitive skill.version_saved dimensions without file path or content", () => {
|
||||
const client = createClient();
|
||||
const dims = {
|
||||
skill_id: "22222222-2222-4222-8222-222222222222",
|
||||
revision_number: 4,
|
||||
file_type: "markdown",
|
||||
skillId: "22222222-2222-4222-8222-222222222222",
|
||||
revisionNumber: 4,
|
||||
fileType: "markdown",
|
||||
file_path: "customers/acme/SKILL.md",
|
||||
file_content: "keep this private",
|
||||
path: "customers/acme/SKILL.md",
|
||||
@@ -269,11 +269,11 @@ describe("shared Skill telemetry proposals", () => {
|
||||
it("emits primitive skill.test_run dimensions without snapshots, prompts, or errors", () => {
|
||||
const client = createClient();
|
||||
const dims = {
|
||||
skill_id: "33333333-3333-4333-8333-333333333333",
|
||||
skillId: "33333333-3333-4333-8333-333333333333",
|
||||
status: "queued",
|
||||
run_source: "run",
|
||||
ad_hoc: true,
|
||||
template_used: false,
|
||||
runSource: "run",
|
||||
adHoc: true,
|
||||
templateUsed: false,
|
||||
inputSnapshot: "do not leak input",
|
||||
outputSnapshot: "do not leak output",
|
||||
harnessIssueDescription: "private harness issue",
|
||||
@@ -302,11 +302,11 @@ describe("shared Skill telemetry proposals", () => {
|
||||
it("emits primitive skill.forked dimensions and passes opaque IDs through raw", () => {
|
||||
const client = createClient();
|
||||
const dims = {
|
||||
skill_id: "44444444-4444-4444-8444-444444444444",
|
||||
fork_from_skill_id: "55555555-5555-4555-8555-555555555555",
|
||||
source_type: "github",
|
||||
sharing_scope: "private",
|
||||
reassign_agent_count: 1,
|
||||
skillId: "44444444-4444-4444-8444-444444444444",
|
||||
forkFromSkillId: "55555555-5555-4555-8555-555555555555",
|
||||
sourceType: "github",
|
||||
sharingScope: "private",
|
||||
reassignAgentCount: 1,
|
||||
name: "Secret Skill",
|
||||
slug: "secret-skill",
|
||||
fork_from_name: "Secret Source",
|
||||
@@ -332,7 +332,7 @@ describe("shared Skill telemetry proposals", () => {
|
||||
it("emits only sharing_scope for the deliberately rejectable share-link proposal", () => {
|
||||
const client = createClient();
|
||||
const dims = {
|
||||
sharing_scope: "public_link",
|
||||
sharingScope: "public_link",
|
||||
skill_id: "66666666-6666-4666-8666-666666666666",
|
||||
href: "https://paperclip.example/skills/secret-skill?token=secret-token",
|
||||
url: "https://paperclip.example/skills/secret-skill?token=secret-token",
|
||||
|
||||
Reference in new issue
Block a user