mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-23 00:53:38 +02:00
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Paperclip records agent run outcomes through telemetry and run lifecycle services > - Terminal run transitions need one consistent event for outcome analysis > - The current paths do not report every terminal transition through one event > - This pull request adds the agent.task_run event and emits it at each terminal transition > - The benefit is complete run outcome data without exposing raw task identifiers ## Linked Issues or Issue Description **What existing behavior does this improve?** Paperclip telemetry reports agent activity, but it does not report every terminal task run through one event. **Subsystem affected** Cross-cutting (multiple of the above): packages/shared telemetry and server run lifecycle services. **Current behavior** Several run paths write a terminal status without a matching agent.task_run telemetry event. **Proposed behavior** Each terminal run transition emits one agent.task_run event. The event records the terminal state and uses the existing pseudonym helper for the optional task identifier. **Reason and benefit** Complete terminal-run data helps operators measure agent outcomes. The pseudonym helper prevents the raw task identifier from leaving the installation. **Breaking changes** None. The change adds an event and keeps existing event behavior compatible. ## What Changed - Add the agent.task_run telemetry contract and client helper. - Reuse the existing pseudonym helper for the task identifier. The helper hashes the identifier with a per-installation salt and returns 16 hexadecimal characters. The raw identifier never leaves the installation. Existing identifiers do not move. - Emit one event from each legacy, native, recovery, and issue terminal transition. - Keep emissions outside database transactions and make delivery best-effort. - Add regression tests for event shape, hashing, terminal transitions, and emission failures. - Document the event and its privacy rule in the telemetry data contract. ## Verification - `npx tsc --noEmit` in `server/` passes at the submitted commit. - The pull-request CI suite must pass. CI is the authority because local Vitest has a known dependency artifact. - The added regression tests cover event output shape, per-installation hash divergence, raw identifier handoff, omitted identifiers, and non-throwing emits. ## Risks - A missed terminal path could reduce event coverage. - Telemetry delivery remains best-effort and cannot change run finalization. - The pseudonym helper uses installation-specific state, so identifiers differ between installations. ## Model Used OpenAI Codex, GPT-5, tool use and code execution. Context window details were not provided. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes: #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I have addressed all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
214 lines
7.8 KiB
TypeScript
214 lines
7.8 KiB
TypeScript
import type { TelemetryClient } from "./client.js";
|
|
import type { EventDimensionsMap } from "./generated/paperclip-telemetry.js";
|
|
|
|
type RawDimension<T extends string | undefined> = T | (string & {});
|
|
|
|
function asEventDimension<T extends string>(value: RawDimension<T>): T {
|
|
return value as T;
|
|
}
|
|
|
|
export function trackInstallStarted(client: TelemetryClient): void {
|
|
client.track("install.started", {});
|
|
}
|
|
|
|
export function trackInstallCompleted(
|
|
client: TelemetryClient,
|
|
dims: { adapterType: RawDimension<EventDimensionsMap["install.completed"]["adapter_type"]> },
|
|
): void {
|
|
client.track("install.completed", {
|
|
adapter_type: asEventDimension(dims.adapterType),
|
|
});
|
|
}
|
|
|
|
export function trackCompanyImported(
|
|
client: TelemetryClient,
|
|
dims: {
|
|
sourceType: RawDimension<EventDimensionsMap["company.imported"]["source_type"]>;
|
|
sourceRef: string;
|
|
isPrivate: boolean;
|
|
},
|
|
): void {
|
|
const ref = dims.isPrivate ? client.hashPrivateRef(dims.sourceRef) : dims.sourceRef;
|
|
client.track("company.imported", {
|
|
source_type: asEventDimension(dims.sourceType),
|
|
source_ref: ref,
|
|
source_ref_hashed: dims.isPrivate,
|
|
});
|
|
}
|
|
|
|
export function trackProjectCreated(client: TelemetryClient): void {
|
|
client.track("project.created", {});
|
|
}
|
|
|
|
export function trackRoutineCreated(client: TelemetryClient): void {
|
|
client.track("routine.created", {});
|
|
}
|
|
|
|
export function trackRoutineRun(
|
|
client: TelemetryClient,
|
|
dims: {
|
|
source: RawDimension<EventDimensionsMap["routine.run"]["source"]>;
|
|
status: RawDimension<EventDimensionsMap["routine.run"]["status"]>;
|
|
},
|
|
): void {
|
|
client.track("routine.run", {
|
|
source: asEventDimension(dims.source),
|
|
status: asEventDimension(dims.status),
|
|
});
|
|
}
|
|
|
|
export function trackGoalCreated(
|
|
client: TelemetryClient,
|
|
dims?: { goalLevel?: RawDimension<EventDimensionsMap["goal.created"]["goal_level"]> | null },
|
|
): void {
|
|
client.track("goal.created", {
|
|
goal_level: dims?.goalLevel ? asEventDimension(dims.goalLevel) : "other",
|
|
});
|
|
}
|
|
|
|
export function trackAgentCreated(
|
|
client: TelemetryClient,
|
|
dims: {
|
|
agentRole: RawDimension<EventDimensionsMap["agent.created"]["agent_role"]>;
|
|
agentId: string;
|
|
},
|
|
): void {
|
|
client.track("agent.created", {
|
|
agent_role: asEventDimension(dims.agentRole),
|
|
agent_id: dims.agentId,
|
|
});
|
|
}
|
|
|
|
export function trackSkillImported(
|
|
client: TelemetryClient,
|
|
dims: {
|
|
sourceType: RawDimension<EventDimensionsMap["skill.imported"]["source_type"]>;
|
|
skillRef?: string | null;
|
|
},
|
|
): void {
|
|
client.track("skill.imported", {
|
|
source_type: asEventDimension(dims.sourceType),
|
|
...(dims.skillRef ? { skill_ref: dims.skillRef } : {}),
|
|
});
|
|
}
|
|
|
|
export function trackAgentFirstHeartbeat(
|
|
client: TelemetryClient,
|
|
dims: {
|
|
agentRole: RawDimension<EventDimensionsMap["agent.first_heartbeat"]["agent_role"]>;
|
|
agentId: string;
|
|
},
|
|
): void {
|
|
client.track("agent.first_heartbeat", {
|
|
agent_role: asEventDimension(dims.agentRole),
|
|
agent_id: dims.agentId,
|
|
});
|
|
}
|
|
|
|
export function trackAgentTaskCompleted(
|
|
client: TelemetryClient,
|
|
dims: {
|
|
agentRole: RawDimension<EventDimensionsMap["agent.task_completed"]["agent_role"]>;
|
|
agentId: string;
|
|
adapterType: RawDimension<EventDimensionsMap["agent.task_completed"]["adapter_type"]>;
|
|
model?: string;
|
|
taskId?: string;
|
|
},
|
|
): void {
|
|
client.track("agent.task_completed", {
|
|
agent_role: asEventDimension(dims.agentRole),
|
|
agent_id: dims.agentId,
|
|
adapter_type: asEventDimension(dims.adapterType),
|
|
...(dims.model ? { model: dims.model } : {}),
|
|
...(dims.taskId ? { task_id: client.hashPrivateRef(dims.taskId) } : {}),
|
|
});
|
|
}
|
|
|
|
export function trackAgentTaskRun(
|
|
client: TelemetryClient,
|
|
dims: {
|
|
agentId: string;
|
|
state: RawDimension<EventDimensionsMap["agent.task_run"]["state"]>;
|
|
adapterType?: RawDimension<EventDimensionsMap["agent.task_run"]["adapter_type"]>;
|
|
agentRole?: RawDimension<EventDimensionsMap["agent.task_run"]["agent_role"]>;
|
|
model?: string;
|
|
durationSeconds?: number;
|
|
inputTokens?: number;
|
|
outputTokens?: number;
|
|
cachedTokens?: number;
|
|
taskId?: string;
|
|
},
|
|
): void {
|
|
client.track("agent.task_run", {
|
|
agent_id: dims.agentId,
|
|
state: asEventDimension(dims.state),
|
|
...(dims.adapterType ? { adapter_type: asEventDimension(dims.adapterType) } : {}),
|
|
...(dims.agentRole ? { agent_role: asEventDimension(dims.agentRole) } : {}),
|
|
...(dims.model ? { model: dims.model } : {}),
|
|
...(dims.durationSeconds === undefined ? {} : { duration_seconds: dims.durationSeconds }),
|
|
...(dims.inputTokens === undefined ? {} : { input_tokens: dims.inputTokens }),
|
|
...(dims.outputTokens === undefined ? {} : { output_tokens: dims.outputTokens }),
|
|
...(dims.cachedTokens === undefined ? {} : { cached_tokens: dims.cachedTokens }),
|
|
...(dims.taskId ? { task_id: client.hashPrivateRef(dims.taskId) } : {}),
|
|
});
|
|
}
|
|
|
|
export function trackErrorHandlerCrash(
|
|
client: TelemetryClient,
|
|
dims: { errorCode: string },
|
|
): void {
|
|
client.track("error.handler_crash", { error_code: dims.errorCode });
|
|
}
|
|
|
|
export function trackInteractionCreated(
|
|
client: TelemetryClient,
|
|
dims: {
|
|
interactionKind: RawDimension<EventDimensionsMap["interaction.created"]["interaction_kind"]>;
|
|
usedDeprecatedResolverPolicyAlias: boolean;
|
|
},
|
|
): void {
|
|
client.track("interaction.created", {
|
|
interaction_kind: asEventDimension(dims.interactionKind),
|
|
used_deprecated_resolver_policy_alias: dims.usedDeprecatedResolverPolicyAlias,
|
|
});
|
|
}
|
|
|
|
export function trackInteractionResolved(
|
|
client: TelemetryClient,
|
|
dims: {
|
|
interactionKind: RawDimension<EventDimensionsMap["interaction.resolved"]["interaction_kind"]>;
|
|
status: RawDimension<EventDimensionsMap["interaction.resolved"]["status"]>;
|
|
resolvedByKind: RawDimension<EventDimensionsMap["interaction.resolved"]["resolved_by_kind"]>;
|
|
resolutionReason?: RawDimension<EventDimensionsMap["interaction.resolved"]["resolution_reason"]> | null;
|
|
createdByKind?: RawDimension<EventDimensionsMap["interaction.resolved"]["created_by_kind"]> | null;
|
|
creatorAgentRole?: RawDimension<EventDimensionsMap["interaction.resolved"]["creator_agent_role"]> | null;
|
|
continuationPolicy?: RawDimension<EventDimensionsMap["interaction.resolved"]["continuation_policy"]> | null;
|
|
targetType?: RawDimension<EventDimensionsMap["interaction.resolved"]["target_type"]> | null;
|
|
optionCount?: number;
|
|
selectedOptionCount?: number;
|
|
questionCount?: number;
|
|
answeredQuestionCount?: number;
|
|
createdTaskCount?: number;
|
|
skippedTaskCount?: number;
|
|
legacyInheritedRestriction: boolean;
|
|
},
|
|
): void {
|
|
client.track("interaction.resolved", {
|
|
interaction_kind: asEventDimension(dims.interactionKind),
|
|
status: asEventDimension(dims.status),
|
|
resolved_by_kind: asEventDimension(dims.resolvedByKind),
|
|
...(dims.resolutionReason ? { resolution_reason: asEventDimension(dims.resolutionReason) } : {}),
|
|
...(dims.createdByKind ? { created_by_kind: asEventDimension(dims.createdByKind) } : {}),
|
|
...(dims.creatorAgentRole ? { creator_agent_role: asEventDimension(dims.creatorAgentRole) } : {}),
|
|
...(dims.continuationPolicy ? { continuation_policy: asEventDimension(dims.continuationPolicy) } : {}),
|
|
...(dims.targetType ? { target_type: asEventDimension(dims.targetType) } : {}),
|
|
...(dims.optionCount === undefined ? {} : { option_count: dims.optionCount }),
|
|
...(dims.selectedOptionCount === undefined ? {} : { selected_option_count: dims.selectedOptionCount }),
|
|
...(dims.questionCount === undefined ? {} : { question_count: dims.questionCount }),
|
|
...(dims.answeredQuestionCount === undefined ? {} : { answered_question_count: dims.answeredQuestionCount }),
|
|
...(dims.createdTaskCount === undefined ? {} : { created_task_count: dims.createdTaskCount }),
|
|
...(dims.skippedTaskCount === undefined ? {} : { skipped_task_count: dims.skippedTaskCount }),
|
|
legacy_inherited_restriction: dims.legacyInheritedRestriction,
|
|
});
|
|
}
|