/**
* What one appended run event costs: statements, wall time, CPU and allocation.
*
* It drives the real append path on a local PostgreSQL:
*
* - `native-stream`: one native run. A runner stream appends its events one
* after another through `PaperclipControlPlanePort.appendEvent`, which is
* what the session runtime calls for every PRP event.
* - `native-burst`: RUNS native runs streaming at the same time, each one in
* order, on one connection pool.
* - `ordinary`: adapter log events through `appendHeartbeatRunEvent`.
*
* Seven of ten native events are text deltas. The other three are progress
* events (`tool.execution.started`, `tool.execution.completed`,
* `item.completed`), which also wake the chat progress queue.
*
* Statements are counted in the driver, so the count includes BEGIN, COMMIT
* and SAVEPOINT. CPU is the whole process (`process.cpuUsage`); busy time is
* the main thread only (event loop utilization). Allocated bytes are V8's
* `total_allocated_bytes` and are omitted on a Node.js without that counter.
*
* TSX_DISABLE_CACHE=1 pnpm --filter @paperclipai/server exec tsx scripts/benchmark-run-event-append.ts
*
* TSX_DISABLE_CACHE keeps tsx from scanning its shared cache directory while
* the first passes run, which otherwise shows up as allocation and CPU here.
*
* To compare with another revision in the same process, in alternating
* passes, write that revision's two modules next to the current ones first:
*
* git show :server/src/services/heartbeat-run-events.ts \
* > server/src/services/heartbeat-run-events.baseline.ts
* git show :server/src/services/native-runtime/paperclip-control-plane-port.ts \
* | sed 's#"../heartbeat-run-events.js"#"../heartbeat-run-events.baseline.js"#' \
* > server/src/services/native-runtime/paperclip-control-plane-port.baseline.ts
*
* Optional: EVENTS (default 2000 per pass and scenario), PASSES (3 per
* implementation), RUNS (8), TEXT_BYTES (96), HISTORY (0 events already
* retained in each run). Delete the baseline copies afterwards.
*/
import assert from "node:assert/strict";
import { randomBytes, randomUUID } from "node:crypto";
import { existsSync } from "node:fs";
import { PerformanceObserver, performance, type PerformanceEntry } from "node:perf_hooks";
import { fileURLToPath } from "node:url";
import v8 from "node:v8";
import vm from "node:vm";
import { eq, sql as drizzleSql } from "drizzle-orm";
import { companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, issues,
startEmbeddedPostgresTestDatabase } from "../../packages/db/src/index.js";
import { createAgentLifecycle } from "../src/services/agent-lifecycle.js";
import * as currentEvents from "../src/services/heartbeat-run-events.js";
import * as currentPort from "../src/services/native-runtime/paperclip-control-plane-port.js";
import type { PrpEvent } from "../src/vendor/paperclip-runner/index.js";
type Implementation = { label: string; events: typeof currentEvents; port: typeof currentPort };
const size = (name: string, fallback: number) => Number(process.env[name] ?? fallback);
const EVENTS = size("EVENTS", 2000), PASSES = size("PASSES", 3), RUNS = size("RUNS", 8);
const TEXT_BYTES = size("TEXT_BYTES", 96), HISTORY = size("HISTORY", 0);
process.env.PAPERCLIP_SECRETS_MASTER_KEY ??= randomBytes(32).toString("base64");
const baselineEvents = fileURLToPath(new URL("../src/services/heartbeat-run-events.baseline.ts", import.meta.url));
const baselinePort = fileURLToPath(new URL("../src/services/native-runtime/paperclip-control-plane-port.baseline.ts", import.meta.url));
const implementations: Implementation[] = [{ label: "current", events: currentEvents, port: currentPort }];
if (existsSync(baselineEvents) && existsSync(baselinePort)) {
implementations.unshift({ label: "baseline", events: await import(baselineEvents), port: await import(baselinePort) });
}
const keepAlive = setInterval(() => {}, 1_000);
const temporary = await startEmbeddedPostgresTestDatabase("run-event-append-");
const db = createDb(temporary.connectionString);
// postgres.js reads this hook for every statement it sends, on every connection.
const sent = { statements: 0, control: 0 };
(db.$client.options as { debug?: unknown }).debug = (_connection: number, text: string) => {
sent.statements += 1;
if (/^\s*(begin|commit|rollback|savepoint|release)\b/i.test(text)) sent.control += 1;
};
// Collections are attributed by their own start time, not by when the
// observer delivers them: delivery is asynchronous and can lag the append.
let collections: PerformanceEntry[] = [];
const gcObserver = new PerformanceObserver(list => { collections.push(...list.getEntries()); });
gcObserver.observe({ entryTypes: ["gc"] });
function collectionsBetween(start: number, end: number) {
collections.push(...gcObserver.takeRecords());
const within = collections.filter(entry => entry.startTime >= start && entry.startTime <= end);
collections = collections.filter(entry => entry.startTime > end);
return { count: within.length, ms: within.reduce((sum, entry) => sum + entry.duration, 0) };
}
const allocated = () => (v8.getHeapStatistics() as { total_allocated_bytes?: number }).total_allocated_bytes;
// A full collection before each pass, without a command-line flag, so that
// garbage from the pass before is not collected inside this one.
v8.setFlagsFromString("--expose-gc");
const collectGarbage = vm.runInNewContext("gc") as () => void;
try {
const [company] = await db.insert(companies).values({ name: "Append", issuePrefix: "APD" }).returning();
const agent = { id: randomUUID() };
await createAgentLifecycle(db).requestHire(company!.id, { id: agent.id, name: "Native", role: "engineer", status: "idle",
adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {} }, { runnerResolved: true });
const [issue] = await db.insert(issues).values({ companyId: company!.id, title: "Streaming task", status: "in_progress",
assigneeAgentId: agent.id }).returning();
const contractSha = "run-event-append-benchmark";
const [contract] = await db.insert(completionContracts).values({ companyId: company!.id, issueId: issue!.id, revision: 1,
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "benchmark", risk: "standard",
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal",
contractJson: { revision: "benchmark", objective: "Benchmark", criteria: [{ id: "objective", requirement: "Append" }] },
canonicalSha256: contractSha, createdByActorType: "system", createdByActorId: "benchmark" }).returning();
const text = randomBytes(Math.ceil(TEXT_BYTES * 0.75)).toString("base64").slice(0, TEXT_BYTES);
async function seedHistory(runId: string, sourceInstanceId: string | null) {
for (let start = 1; start <= HISTORY; start += 500) {
await db.insert(heartbeatRunEvents).values(Array.from({ length: Math.min(500, HISTORY - start + 1) }, (_, index) => ({
companyId: company!.id, agentId: agent.id, runId, seq: start + index, eventType: "item.delta",
...(sourceInstanceId ? { sourceInstanceId, sourceSeq: start + index, sourceEventId: `${sourceInstanceId}:${start + index}` } : {}),
})));
}
if (HISTORY) await db.update(heartbeatRuns).set({ nextEventSeq: HISTORY + 1 }).where(eq(heartbeatRuns.id, runId));
}
async function nativeRun(implementation: Implementation) {
const runId = randomUUID(), sessionId = randomUUID(), runnerId = randomUUID();
await db.insert(heartbeatRuns).values({ id: runId, companyId: company!.id, agentId: agent.id, status: "running",
runtimeMode: "native", nativeIssueId: issue!.id, issueId: issue!.id, nativeSessionId: sessionId, runnerInstanceId: runnerId,
completionContractId: contract!.id, completionContractSha256: contractSha, contextSnapshot: { issueId: issue!.id } });
await seedHistory(runId, runnerId);
const identity = { companyId: company!.id, issueId: issue!.id, runId, agentId: agent.id, sessionId };
const port = new implementation.port.PaperclipControlPlanePort(db, { ...identity, completionContractId: contract!.id,
completionContractSha256: contractSha, sourceInstanceId: runnerId, controlPlaneSourceInstanceId: `control-${runId}` });
await port.openRun({ identity, backendKind: "mock", sourceInstanceId: runnerId } as Parameters[0]);
const event = (ordinal: number): PrpEvent => {
const sourceSeq = HISTORY + ordinal, step = ordinal % 10;
const base = { schema: "paperclip.prp.event.v1" as const, sourceEventId: `${runnerId}:${sourceSeq}`, sourceSeq,
sourceInstanceId: runnerId, sourceKind: "runner" as const, runId, normalizedSessionId: sessionId,
turnId: "turn-1", schemaVersion: 1 as const, priority: 1, emittedAt: new Date().toISOString() };
if (step < 7) return { ...base, itemId: "message-1", eventType: "item.delta", payload: { kind: "agentMessage", text } } as PrpEvent;
if (step === 9) return { ...base, itemId: `command-${ordinal}`, eventType: "item.completed",
payload: { kind: "commandExecution", item: { type: "command", id: `command-${ordinal}`, output: text } } } as PrpEvent;
return { ...base, itemId: `tool-${ordinal}`, eventType: step === 7 ? "tool.execution.started" : "tool.execution.completed",
payload: { schema: "paperclip.tool.execution.v1", executionId: `tool-${ordinal - (step === 7 ? 0 : 1)}`, transport: "builtin",
operation: "execute", name: "shell", target: null, namespace: null, readOnly: true,
status: step === 7 ? "running" : "completed", durationMs: step === 7 ? null : 12, exitCode: step === 7 ? null : 0,
progress: null, output: text, outputBytes: text.length, outputTruncated: false,
outputDigest: `sha256:${"a".repeat(64)}` } } as PrpEvent;
};
return { runId, append: async (count: number) => {
for (let ordinal = 1; ordinal <= count; ordinal++) {
const receipt = await port.appendEvent(event(ordinal));
assert.equal(receipt.disposition, "committed");
assert.equal(receipt.highestContiguousSourceSeq, HISTORY + ordinal);
}
} };
}
async function ordinaryRun(implementation: Implementation) {
const runId = randomUUID();
await db.insert(heartbeatRuns).values({ id: runId, companyId: company!.id, agentId: agent.id, status: "running" });
await seedHistory(runId, null);
return { runId, append: async (count: number) => {
for (let ordinal = 1; ordinal <= count; ordinal++) {
const receipt = await implementation.events.appendHeartbeatRunEvent(db, { companyId: company!.id, runId, agentId: agent.id,
eventType: "adapter.output", stream: "stdout", level: "info", message: text, payload: { chunk: ordinal } });
assert.equal(receipt.row.seq, HISTORY + ordinal);
}
} };
}
const scenarios = {
"native-stream": async (implementation: Implementation) => [{ run: await nativeRun(implementation), events: EVENTS }],
"native-burst": async (implementation: Implementation) => Promise.all(Array.from({ length: RUNS }, async () =>
({ run: await nativeRun(implementation), events: Math.ceil(EVENTS / RUNS) }))),
ordinary: async (implementation: Implementation) => [{ run: await ordinaryRun(implementation), events: EVENTS }],
};
async function measure(name: keyof typeof scenarios, implementation: Implementation) {
const streams = await scenarios[name](implementation);
const events = streams.reduce((sum, stream) => sum + stream.events, 0);
collectGarbage();
const before = { statements: sent.statements, control: sent.control, cpu: process.cpuUsage(), loop: performance.eventLoopUtilization(),
bytes: allocated(), at: performance.now() };
await Promise.all(streams.map(stream => stream.run.append(stream.events)));
const endedAt = performance.now(), wallMs = endedAt - before.at, cpu = process.cpuUsage(before.cpu);
const loop = performance.eventLoopUtilization(before.loop), bytes = allocated();
const statements = sent.statements - before.statements, control = sent.control - before.control;
// Every measurement stops at endedAt, before the stored rows are read
// back. Node.js reports a collection from a later immediate, so wait one
// turn before draining; only collections that began in the window count.
await new Promise(resolve => setImmediate(resolve));
const gc = collectionsBetween(before.at, endedAt);
for (const stream of streams) {
const rows = await db.select({ seq: heartbeatRunEvents.seq }).from(heartbeatRunEvents)
.where(eq(heartbeatRunEvents.runId, stream.run.runId)).orderBy(heartbeatRunEvents.seq);
assert.deepEqual(rows.map(row => row.seq), Array.from({ length: HISTORY + stream.events }, (_, index) => index + 1),
"Every event is stored once, in order, without a gap");
}
return { events, statementsPerEvent: statements / events, transactionControlPerEvent: control / events,
wallMsPerEvent: wallMs / events, cpuMsPerEvent: (cpu.user + cpu.system) / 1000 / events,
mainThreadBusyMsPerEvent: loop.active / events,
allocatedKbPerEvent: bytes === undefined || before.bytes === undefined ? null : (bytes - before.bytes) / 1024 / events,
gcPausesPer1000Events: gc.count * 1000 / events, gcMsPer1000Events: gc.ms * 1000 / events };
}
const names = Object.keys(scenarios) as Array;
// One discarded pass per implementation warms the JIT and the connection pool.
for (const implementation of implementations) for (const name of names) await measure(name, implementation);
const samples = new Map>>>();
for (let pass = 0; pass < PASSES; pass++) {
// Alternate the order so machine load drifts over both implementations alike.
for (const implementation of pass % 2 === 0 ? implementations : [...implementations].reverse()) {
for (const name of names) {
const key = `${name}/${implementation.label}`;
samples.set(key, [...samples.get(key) ?? [], await measure(name, implementation)]);
}
}
}
const median = (values: number[]) => [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]!;
const round = (value: number) => Number(value.toFixed(3));
const results = names.flatMap(name => implementations.map(implementation => {
const passes = samples.get(`${name}/${implementation.label}`)!;
const field = (key: keyof (typeof passes)[number]) => passes.every(pass => pass[key] !== null)
? round(median(passes.map(pass => pass[key] as number))) : null;
return { scenario: name, implementation: implementation.label, events: passes[0]!.events,
statementsPerEvent: field("statementsPerEvent"), transactionControlPerEvent: field("transactionControlPerEvent"),
wallMsPerEvent: field("wallMsPerEvent"), cpuMsPerEvent: field("cpuMsPerEvent"),
mainThreadBusyMsPerEvent: field("mainThreadBusyMsPerEvent"), allocatedKbPerEvent: field("allocatedKbPerEvent"),
gcPausesPer1000Events: field("gcPausesPer1000Events"), gcMsPer1000Events: field("gcMsPer1000Events") };
}));
const [{ version }] = await db.execute(drizzleSql`select version() as version`) as unknown as [{ version: string }];
console.log(JSON.stringify({ node: process.version, postgres: version.split(" ").slice(0, 2).join(" "),
passes: PASSES, runs: RUNS, textBytes: TEXT_BYTES, history: HISTORY, results }, null, 2));
} finally {
gcObserver.disconnect();
clearInterval(keepAlive);
await temporary.cleanup();
}