mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 16:27:18 +02:00
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Paperclip records agent run outcomes through telemetry and run lifecycle services > - Terminal run transitions need one consistent event for outcome analysis > - The current paths do not report every terminal transition through one event > - This pull request adds the agent.task_run event and emits it at each terminal transition > - The benefit is complete run outcome data without exposing raw task identifiers ## Linked Issues or Issue Description **What existing behavior does this improve?** Paperclip telemetry reports agent activity, but it does not report every terminal task run through one event. **Subsystem affected** Cross-cutting (multiple of the above): packages/shared telemetry and server run lifecycle services. **Current behavior** Several run paths write a terminal status without a matching agent.task_run telemetry event. **Proposed behavior** Each terminal run transition emits one agent.task_run event. The event records the terminal state and uses the existing pseudonym helper for the optional task identifier. **Reason and benefit** Complete terminal-run data helps operators measure agent outcomes. The pseudonym helper prevents the raw task identifier from leaving the installation. **Breaking changes** None. The change adds an event and keeps existing event behavior compatible. ## What Changed - Add the agent.task_run telemetry contract and client helper. - Reuse the existing pseudonym helper for the task identifier. The helper hashes the identifier with a per-installation salt and returns 16 hexadecimal characters. The raw identifier never leaves the installation. Existing identifiers do not move. - Emit one event from each legacy, native, recovery, and issue terminal transition. - Keep emissions outside database transactions and make delivery best-effort. - Add regression tests for event shape, hashing, terminal transitions, and emission failures. - Document the event and its privacy rule in the telemetry data contract. ## Verification - `npx tsc --noEmit` in `server/` passes at the submitted commit. - The pull-request CI suite must pass. CI is the authority because local Vitest has a known dependency artifact. - The added regression tests cover event output shape, per-installation hash divergence, raw identifier handoff, omitted identifiers, and non-throwing emits. ## Risks - A missed terminal path could reduce event coverage. - Telemetry delivery remains best-effort and cannot change run finalization. - The pseudonym helper uses installation-specific state, so identifiers differ between installations. ## Model Used OpenAI Codex, GPT-5, tool use and code execution. Context window details were not provided. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes: #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I have addressed all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
125 lines
4.0 KiB
TypeScript
125 lines
4.0 KiB
TypeScript
import { eq } from "drizzle-orm";
|
|
import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
|
|
import { trackAgentTaskRun } from "@paperclipai/shared/telemetry";
|
|
import { parseObject } from "../adapters/utils.js";
|
|
import { logger } from "../middleware/logger.js";
|
|
import { getTelemetryClient } from "../telemetry.js";
|
|
|
|
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
|
|
|
|
function readNonEmptyString(value: unknown): string | null {
|
|
return typeof value === "string" && value.trim().length > 0
|
|
? value.trim()
|
|
: null;
|
|
}
|
|
|
|
function readUsageToken(
|
|
source: Record<string, unknown>,
|
|
...keys: string[]
|
|
): number {
|
|
for (const key of keys) {
|
|
const value = source[key];
|
|
if (typeof value === "number" && Number.isFinite(value)) {
|
|
return Math.max(0, Math.floor(value));
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
function resolveDurationSeconds(
|
|
run: HeartbeatRun,
|
|
now: Date,
|
|
): number | undefined {
|
|
if (!run.startedAt) return undefined;
|
|
const startedAtMs = new Date(run.startedAt).getTime();
|
|
const endMs = run.finishedAt ? new Date(run.finishedAt).getTime() : now.getTime();
|
|
return Math.max(0, Math.round((endMs - startedAtMs) / 1000));
|
|
}
|
|
|
|
/**
|
|
* Emits one `agent.task_run` telemetry event for a run that a caller already
|
|
* wrote to a terminal status. The native runtime and the legacy run engine
|
|
* both call this function beside each terminal status write. Call this
|
|
* function after the run commits.
|
|
*
|
|
* This function never throws. A telemetry failure must never fail a run.
|
|
* Telemetry enrichment does an awaited database lookup, so treat this call
|
|
* as best-effort background work: do not let a caller's own required
|
|
* lifecycle work (publishing a live event, cancelling a wake, clearing an
|
|
* issue lock, returning a response) wait behind it. Fire this call with
|
|
* `void emitAgentTaskRun(...)` and never await it, even alongside other
|
|
* required work with `Promise.all` — an awaited `Promise.all` still blocks
|
|
* the caller's return on the slower of the two promises, so it does not
|
|
* remove the delay.
|
|
*/
|
|
export async function emitAgentTaskRun(db: Db, run: HeartbeatRun): Promise<void> {
|
|
try {
|
|
const client = getTelemetryClient();
|
|
if (!client) return;
|
|
|
|
const agent = await db
|
|
.select({
|
|
role: agents.role,
|
|
adapterType: agents.adapterType,
|
|
adapterConfig: agents.adapterConfig,
|
|
})
|
|
.from(agents)
|
|
.where(eq(agents.id, run.agentId))
|
|
.then((rows) => rows[0] ?? null);
|
|
|
|
const usage = parseObject(run.usageJson);
|
|
const model =
|
|
readNonEmptyString(usage.model) ??
|
|
readNonEmptyString(parseObject(agent?.adapterConfig).model) ??
|
|
undefined;
|
|
|
|
const taskId =
|
|
readNonEmptyString(parseObject(run.contextSnapshot).issueId) ??
|
|
undefined;
|
|
|
|
trackAgentTaskRun(client, {
|
|
agentId: run.agentId,
|
|
state: run.status,
|
|
...(agent
|
|
? { adapterType: agent.adapterType, agentRole: agent.role }
|
|
: {}),
|
|
...(model ? { model } : {}),
|
|
...(() => {
|
|
const durationSeconds = resolveDurationSeconds(run, new Date());
|
|
return durationSeconds === undefined ? {} : { durationSeconds };
|
|
})(),
|
|
...(run.usageJson != null
|
|
? {
|
|
inputTokens: readUsageToken(
|
|
usage,
|
|
"inputTokens",
|
|
"input_tokens",
|
|
"rawInputTokens",
|
|
"raw_input_tokens",
|
|
),
|
|
outputTokens: readUsageToken(
|
|
usage,
|
|
"outputTokens",
|
|
"output_tokens",
|
|
"rawOutputTokens",
|
|
"raw_output_tokens",
|
|
),
|
|
cachedTokens: readUsageToken(
|
|
usage,
|
|
"cachedInputTokens",
|
|
"cached_input_tokens",
|
|
"cacheReadInputTokens",
|
|
"cache_read_input_tokens",
|
|
),
|
|
}
|
|
: {}),
|
|
...(taskId ? { taskId } : {}),
|
|
});
|
|
} catch (err) {
|
|
logger.warn(
|
|
{ err, runId: run.id },
|
|
"failed to emit agent.task_run telemetry",
|
|
);
|
|
}
|
|
}
|