Files
PaperClipAI/server/src/services/agent-task-run-telemetry.ts
T
Nicky LeachandPaperclip f65991a5f1 refactor(server): move scheduled-retry and queued-run dispatch into a run-dispatch module (#12920)
## Thinking Path

> - Paperclip is the open source app people use to manage AI agents for
work.
> - The server heartbeat service dispatches scheduled retries and queued
runs.
> - The service kept policy decisions and database writes in one large
file.
> - This layout made policy branches harder to test and transaction
boundaries harder to inspect.
> - This pull request moves the policy rules and database transactions
into a run-dispatch module.
> - The benefit is a smaller service, pure policy tests, and clear
transaction ownership.

## Linked Issues or Issue Description

**What existing behavior does this improve?**

The server heartbeat service promotes scheduled retries and cancels
stale queued runs.

**Subsystem affected**

server/ — REST API and orchestration services.

**Current behavior**

The heartbeat service contains the policy rules and the database writes
for these dispatch paths.

**Proposed behavior**

A run-dispatch module owns pure policy functions and semantic database
transactions. The public service contracts stay unchanged.

**Reason and benefit**

The new layout separates branch rules from database effects. It makes
each policy branch easier to test and keeps each operation’s row writes
in one transaction.

**Breaking changes**

None. The public service contracts stay unchanged.

## What Changed

- Move scheduled-retry promotion and queued-run staleness rules into
pure functions.
- Add table-driven unit tests for each policy branch.
- Move promotion and cancellation writes into semantic transactions.
- Keep row locking, company isolation, and post-commit effects
unchanged.

## Verification

- `node scripts/check-module-boundaries.mjs` passes.
- `tsc --noEmit` from `server/` reports no errors.
- The focused server test command passes 228 tests in six files.
- Full pull request CI passes.
- Greptile reports 5/5, and all review threads are resolved.

## Risks

The main risk concerns changed transaction boundaries in scheduled-retry
promotion and queued-run cancellation. The focused tests retain coverage
for locking, transactionality, company isolation, and transport
contracts. The public service contracts do not change.

## Model Used

Codex, GPT-5, with code execution and tool use. The context window is
not provided by the runtime.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either (a) linked existing issues with `Fixes: #` /
`Closes: #` / `Refs: #` OR (b) described the issue in-PR following the
relevant issue template
- [x] I have not referenced internal/instance-local Paperclip issues or
links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip`
URLs)
- [x] My branch name describes the change (e.g. `docs/...`, `fix/...`)
and contains no internal Paperclip ticket id or instance-derived details
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] All Paperclip CI gates are green
- [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups
- [x] I have addressed all Greptile and reviewer comments before
requesting merge

---------

Co-authored-by: Paperclip <noreply@paperclip.ing>
2026-09-08 11:00:55 -07:00

142 lines
4.6 KiB
TypeScript

import { and, eq } from "drizzle-orm";
import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
import { trackAgentTaskRun } from "@paperclipai/shared/telemetry";
import { parseObject } from "../adapters/utils.js";
import { logger } from "../middleware/logger.js";
import { getTelemetryClient } from "../telemetry.js";
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
function readNonEmptyString(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0
? value.trim()
: null;
}
function readUsageToken(
source: Record<string, unknown>,
...keys: string[]
): number {
for (const key of keys) {
const value = source[key];
if (typeof value === "number" && Number.isFinite(value)) {
return Math.max(0, Math.floor(value));
}
}
return 0;
}
function resolveDurationSeconds(
run: HeartbeatRun,
now: Date,
): number | undefined {
if (!run.startedAt) return undefined;
const startedAtMs = new Date(run.startedAt).getTime();
const endMs = run.finishedAt ? new Date(run.finishedAt).getTime() : now.getTime();
return Math.max(0, Math.round((endMs - startedAtMs) / 1000));
}
/**
* Emits one `agent.task_run` telemetry event for a run that a caller already
* wrote to a terminal status. The native runtime and the legacy run engine
* both call this function beside each terminal status write. Call this
* function after the run commits.
*
* This function never throws. A telemetry failure must never fail a run.
* Telemetry enrichment does an awaited database lookup, so treat this call
* as best-effort background work: do not let a caller's own required
* lifecycle work (publishing a live event, cancelling a wake, clearing an
* issue lock, returning a response) wait behind it. Fire this call with
* `void emitAgentTaskRun(...)` and never await it, even alongside other
* required work with `Promise.all` — an awaited `Promise.all` still blocks
* the caller's return on the slower of the two promises, so it does not
* remove the delay.
*/
export async function emitAgentTaskRun(db: Db, run: HeartbeatRun): Promise<void> {
try {
const client = getTelemetryClient();
if (!client) return;
const agent = await db
.select({
role: agents.role,
adapterType: agents.adapterType,
adapterConfig: agents.adapterConfig,
})
.from(agents)
.where(eq(agents.id, run.agentId))
.then((rows) => rows[0] ?? null);
const usage = parseObject(run.usageJson);
const model =
readNonEmptyString(usage.model) ??
readNonEmptyString(parseObject(agent?.adapterConfig).model) ??
undefined;
const taskId =
readNonEmptyString(parseObject(run.contextSnapshot).issueId) ??
undefined;
trackAgentTaskRun(client, {
agentId: run.agentId,
state: run.status,
...(agent
? { adapterType: agent.adapterType, agentRole: agent.role }
: {}),
...(model ? { model } : {}),
...(() => {
const durationSeconds = resolveDurationSeconds(run, new Date());
return durationSeconds === undefined ? {} : { durationSeconds };
})(),
...(run.usageJson != null
? {
inputTokens: readUsageToken(
usage,
"inputTokens",
"input_tokens",
"rawInputTokens",
"raw_input_tokens",
),
outputTokens: readUsageToken(
usage,
"outputTokens",
"output_tokens",
"rawOutputTokens",
"raw_output_tokens",
),
cachedTokens: readUsageToken(
usage,
"cachedInputTokens",
"cached_input_tokens",
"cacheReadInputTokens",
"cache_read_input_tokens",
),
}
: {}),
...(taskId ? { taskId } : {}),
});
} catch (err) {
logger.warn(
{ err, runId: run.id },
"failed to emit agent.task_run telemetry",
);
}
}
/** Loads a committed run before emitting telemetry, keeping database rows out of application contracts. */
export async function emitAgentTaskRunById(
db: Db,
input: { runId: string; companyId: string },
): Promise<void> {
try {
const run = await db
.select()
.from(heartbeatRuns)
.where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId)))
.then((rows) => rows[0] ?? null);
if (run) await emitAgentTaskRun(db, run);
} catch (err) {
logger.warn({ err, runId: input.runId }, "failed to load run for agent.task_run telemetry");
}
}