/**
* Local proof for the run-event append lock and controller checkpoint saves.
*
* A native run appends every PRP event through `appendHeartbeatRunEvent`,
* which locks the run row. The controller then saves its checkpoint: a new
* `event:N` row plus the `cursor` row. The new row's foreign key check takes
* FOR KEY SHARE on the same run row. This script replays that pattern with the
* real append and the real checkpoint writer, on separate connections, and
* times each checkpoint save. It compares a baseline append module (FOR UPDATE)
* with the current one (FOR NO KEY UPDATE), in alternating order.
*
* Overlap is the normal controller order. The frame chain awaits each event's
* append (onCommittedEvent) and local commit, but not its checkpoint save: the
* ACK goes through #sendJson, which queues the save and does not block the
* next frame. Only semantic inputs and command results await the save. So the
* save for event N runs while event N+1 is appended. SERIAL=1 awaits every
* save before the next append, as a control without overlap.
*
* HOLD_MS keeps each append transaction open that much longer. It stands in
* for hosted round trips, which make the real append transaction last longer
* than on loopback.
*
* git show :server/src/services/heartbeat-run-events.ts \
* > server/src/services/heartbeat-run-events.baseline.ts
* pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-run-event-checkpoint-lock.ts
*
* Optional: EVENTS (default 400 per pass), PASSES (4), HOLD_MS (5), EVENT_KIB (2), SERIAL (0).
* Delete the baseline copy afterwards.
*/
import assert from "node:assert/strict";
import { randomBytes, randomUUID } from "node:crypto";
import { existsSync } from "node:fs";
import { createRequire } from "node:module";
import { fileURLToPath } from "node:url";
import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations,
nativeRunnerCheckpoints, startEmbeddedPostgresTestDatabase } from "../packages/db/src/index.js";
import * as candidate from "../server/src/services/heartbeat-run-events.js";
import { nativeRunnerCheckpointWriter } from "../server/src/services/native-runtime/native-runner-checkpoint.js";
const { eq, sql: drizzleSql } = createRequire(new URL("../server/package.json", import.meta.url))("drizzle-orm") as typeof import("drizzle-orm");
const baselinePath = fileURLToPath(new URL("../server/src/services/heartbeat-run-events.baseline.ts", import.meta.url));
assert(existsSync(baselinePath), "Write the baseline module first (see the header)");
const baseline = await import(baselinePath) as typeof candidate;
const size = (name: string, fallback: number) => Number(process.env[name] ?? fallback);
const EVENTS = size("EVENTS", 400), PASSES = size("PASSES", 4), HOLD_MS = size("HOLD_MS", 5), EVENT_KIB = size("EVENT_KIB", 2);
const SERIAL = process.env.SERIAL === "1";
process.env.PAPERCLIP_SECRETS_MASTER_KEY ??= randomBytes(32).toString("base64");
const keepAlive = setInterval(() => {}, 1_000);
const temporary = await startEmbeddedPostgresTestDatabase("run-event-checkpoint-lock-");
const appendDb = createDb(temporary.connectionString);
const checkpointDb = createDb(temporary.connectionString);
try {
const [company] = await appendDb.insert(companies).values({ name: "Lock", issuePrefix: "LCK" }).returning();
const [agent] = await appendDb.insert(agents).values({ companyId: company!.id, name: "Native" }).returning();
const [issue] = await appendDb.insert(issues).values({ companyId: company!.id, title: "Streaming task" }).returning();
const quantile = (values: number[], q: number) => [...values].sort((a, b) => a - b)[Math.min(values.length - 1, Math.floor(values.length * q))]!;
const round = (value: number) => Number(value.toFixed(2));
async function pass(label: "baseline" | "candidate") {
const append = (label === "baseline" ? baseline : candidate).appendHeartbeatRunEvent;
const runId = randomUUID(), bootId = randomUUID(), leaseOwner = randomUUID();
await appendDb.insert(heartbeatRuns).values({ id: runId, companyId: company!.id, agentId: agent!.id, status: "running",
runtimeMode: "native", nativeIssueId: issue!.id, issueId: issue!.id,
runnerProfileJson: { padding: randomBytes(3 * 1024).toString("base64") } });
await appendDb.insert(nativeRunFinalizations).values({ runId, companyId: company!.id, issueId: issue!.id, phase: "observed",
controllerBootId: bootId, controllerGeneration: 1, leaseOwner, leaseExpiresAt: new Date(Date.now() + 3_600_000) });
const write = nativeRunnerCheckpointWriter(checkpointDb, { companyId: company!.id, issueId: issue!.id, runId,
authorityRunId: runId, bootId, generation: 1, leaseOwner });
const checkpoint = (records: Map, changed: Map, reset: boolean) => ({
runId, records, changedRecords: changed, removedRecords: [], reset, workCommandIds: [], predecessor: null,
runnerVersion: "bench", runnerDigest: `sha256:${"a".repeat(64)}`,
recoveryEvidence: { runnerInstanceId: "bench", normalizedSessionId: "bench", authenticatedConnectionCount: 1, highestRecordedSourceSeq: 0 },
restoreSessionDirectory() {},
}) as unknown as Parameters[0];
const records = new Map([["header", JSON.stringify({ padding: randomBytes(6 * 1024).toString("base64") })],
["cursor", JSON.stringify({ ackedSourceSeq: 0 })]]);
await write(checkpoint(records, new Map(records), true));
const saves: number[] = [];
let tail = Promise.resolve();
const started = performance.now();
for (let seq = 1; seq <= EVENTS; seq++) {
const event = { sourceSeq: seq, eventType: "item.completed", text: randomBytes(EVENT_KIB * 768).toString("base64") };
// The append commits before the controller ACKs, like onCommittedEvent.
await appendDb.transaction(async tx => {
await append(tx as unknown as typeof appendDb, { companyId: company!.id, agentId: agent!.id, runId,
eventType: "item.completed", stream: "system", level: "info", payload: { prpEvent: event },
nativeSource: { sourceInstanceId: "bench-runner", sourceEventId: `event-${seq}`, sourceSeq: seq,
protocolSchemaVersion: 1, canonicalPayload: event } });
if (HOLD_MS) await tx.execute(drizzleSql`select pg_sleep(${HOLD_MS / 1000})`);
});
const changed = new Map([[`event:${seq}`, JSON.stringify(event)], ["cursor", JSON.stringify({ ackedSourceSeq: seq })]]);
for (const [key, value] of changed) records.set(key, value);
// Checkpoint saves are serialized per controller and overlap the next append.
tail = tail.then(async () => {
const start = performance.now();
await write(checkpoint(new Map(records), changed, false));
saves.push(performance.now() - start);
});
if (SERIAL) await tail;
}
await tail;
const elapsed = performance.now() - started;
const [stored] = await checkpointDb.select({ count: drizzleSql`count(*)::int` }).from(nativeRunnerCheckpoints)
.where(eq(nativeRunnerCheckpoints.runId, runId));
const events = await appendDb.select({ seq: heartbeatRunEvents.seq }).from(heartbeatRunEvents)
.where(eq(heartbeatRunEvents.runId, runId)).orderBy(heartbeatRunEvents.seq);
assert.equal(stored!.count, EVENTS + 2, "Every event row, the header and the cursor are stored");
assert.deepEqual(events.map(row => row.seq), Array.from({ length: EVENTS }, (_, index) => index + 1));
return { label, saves: saves.length, saveMedianMs: round(quantile(saves, 0.5)), saveP90Ms: round(quantile(saves, 0.9)),
saveMaxMs: round(Math.max(...saves)), savesOver2Ms: saves.filter(value => value > 2).length,
saveTotalMs: round(saves.reduce((sum, value) => sum + value, 0)), elapsedMs: round(elapsed) };
}
await pass("candidate"); // Warm caches and the prepared-statement path once.
const results = [];
for (let index = 0; index < PASSES; index++) results.push(await pass(index % 2 === 0 ? "baseline" : "candidate"));
console.log(JSON.stringify({ events: EVENTS, serial: SERIAL, holdMs: HOLD_MS, eventKiB: EVENT_KIB, results }, null, 2));
} finally {
clearInterval(keepAlive);
await temporary.cleanup();
}