diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-checkpoint-records.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-checkpoint-records.test.ts new file mode 100644 index 00000000000..bba2f41a3c6 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/durable-prp-checkpoint-records.test.ts @@ -0,0 +1,187 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; + +import { afterEach, expect, it, vi } from "vitest"; + +import { + DurablePrpControlPlane, + decodeDurablePrpCheckpoint, + durableRecoveryInternals, + type DurablePrpCheckpoint, + type DurablePrpControlPlaneStore, +} from "./durable-prp-control-plane.js"; +import type { DurableRecoveryIdentity } from "./prp-transport-types.js"; + +type CoreState = DurablePrpControlPlaneStore["state"]; +type CoreCommand = CoreState["commands"][number]; +type CoreEvent = CoreState["committedEvents"][number]; + +const identity: DurableRecoveryIdentity = { + runnerInstanceId: "runner-checkpoint-records-1", + environmentLeaseId: "environment-checkpoint-records-1", + runId: "00000000-0000-4000-8000-000000000101", + normalizedSessionId: "session-checkpoint-records-1", + turnId: "turn-checkpoint-records-1", + itemId: "item-checkpoint-records-1", +}; +const runnerVersion = "0.3.0"; +const runnerDigest = `sha256:${"a".repeat(64)}`; +const roots: string[] = []; + +afterEach(() => { + vi.restoreAllMocks(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function temporaryRoot(): string { + const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-records-")); + roots.push(root); + return root; +} + +function command(controllerSeq: number, status: CoreCommand["status"] = "completed"): CoreCommand { + return { + schema: "paperclip.prp.command.v1", commandId: `command-${controllerSeq}`, controllerSeq, + type: "session.snapshot", issuedAt: "2026-11-13T00:00:00.000Z", + payload: { request: controllerSeq, text: "x".repeat(64) }, + status, result: status === "pending" ? null : { snapshot: `snapshot-${controllerSeq}` }, + }; +} + +function event(sourceSeq: number, eventType = "item.completed"): CoreEvent { + const streaming = eventType === "item.delta" || eventType === "usage.reported"; + return { + sourceSeq, sourceEventId: `event-${sourceSeq}`, eventType, priority: streaming ? 2 : 1, + envelope: { kind: "event", runId: identity.runId, payload: { sourceSeq, eventType, text: "y".repeat(128) } }, + deliveryCount: 1, logicalEffectCount: 1, + }; +} + +function entries(checkpoint: DurablePrpCheckpoint): Array<[string, string]> { + return [...checkpoint.records]; +} + +it("serialises only the new command when one command is added to a full journal", async () => { + const root = temporaryRoot(); + const options = { stateDirectory: resolve(root, "control-plane"), identity, + expectedRunnerVersion: runnerVersion, expectedRunnerDigest: runnerDigest }; + const core = new DurablePrpControlPlane(options); + const commands = Array.from({ length: 499 }, (_, index) => command(index + 1)); + const events = Array.from({ length: 4_000 }, (_, index) => event(index + 1)); + const working = core.store.state; + working.commands = commands; + working.committedEvents = events; + working.ackedSourceSeq = events.length; + (core.store as DurablePrpControlPlaneStore & { save(): void }).save(); + const saved: DurablePrpCheckpoint[] = []; + core.setCheckpointPersistence(async checkpoint => { saved.push(checkpoint); }); + let restored: DurablePrpControlPlane | undefined; + try { + await core.flushCheckpoint(); + const existing = new Set([...commands, ...events]); + const stringify = vi.spyOn(JSON, "stringify"); + core.queueCommand("session.snapshot", {}, "probe-command"); + await core.flushCheckpoint(); + const recordCalls = stringify.mock.calls.filter(([value]) => value !== null && typeof value === "object" + && ("controllerSeq" in value || "sourceSeq" in value)); + stringify.mockRestore(); + // The new command is serialised for its size check and once for the checkpoint. + expect(recordCalls.filter(([value]) => existing.has(value as object))).toHaveLength(0); + expect(recordCalls.length).toBeLessThanOrEqual(2); + + const checkpoint = saved.at(-1)!; + expect(checkpoint.reset).toBe(false); + expect([...checkpoint.changedRecords.keys()].filter(key => key.startsWith("command:") || key.startsWith("event:"))) + .toEqual(["command:500"]); + expect(checkpoint.removedRecords).toEqual([]); + + await core.stop(); + rmSync(options.stateDirectory, { recursive: true }); + decodeDurablePrpCheckpoint(new Map(checkpoint.records)).restoreSessionDirectory(root); + restored = new DurablePrpControlPlane(options); + expect(restored.store.state.commands).toHaveLength(500); + expect(restored.store.state.committedEvents).toHaveLength(4_000); + expect(restored.getCommand("probe-command")).toMatchObject({ controllerSeq: 500, status: "pending" }); + // The restored journal produces byte-identical records without any cache. + expect(entries(durableRecoveryInternals.captureCheckpoint(restored.store.state, runnerVersion, runnerDigest))) + .toEqual(entries(checkpoint)); + } finally { + await restored?.stop(); + await core.stop(); + } +}); + +/** Small deterministic generator (mulberry32), so a failure is reproducible. */ +function random(seed: number): () => number { + let value = seed >>> 0; + return () => { + value = (value + 0x6d2b79f5) >>> 0; + let mixed = Math.imul(value ^ (value >>> 15), 1 | value); + mixed = (mixed + Math.imul(mixed ^ (mixed >>> 7), 61 | mixed)) ^ mixed; + return ((mixed ^ (mixed >>> 14)) >>> 0) / 4_294_967_296; + }; +} + +it.each([1, 2, 3, 4, 5])("produces the same records as a full serialisation across varied journal edits (seed %i)", seed => { + const core = new DurablePrpControlPlane({ stateDirectory: temporaryRoot(), identity, + expectedRunnerVersion: runnerVersion, expectedRunnerDigest: runnerDigest }); + let state: CoreState = structuredClone(core.store.state); + const next = random(seed); + const pick = (values: readonly T[]): T | undefined => values[Math.floor(next() * values.length)]; + let sourceSeq = 0; + const eventTypes = ["item.completed", "item.delta", "usage.reported", "turn.started", "turn.completed", "harness.ready"]; + const operations: Array<() => void> = [ + () => { if (state.commands.length < 500) state.commands.push(command(state.commands.length + 1, "pending")); }, + () => { + const pending = pick(state.commands.filter(candidate => candidate.status === "pending")); + if (pending) { pending.status = pick(["completed", "failed", "rejected"] as const)!; pending.result = { value: next() }; } + }, + () => { + if (state.commands.length === 0) return; + state.commands.splice(Math.floor(next() * state.commands.length), 1); + state.commands.forEach((candidate, index) => { candidate.controllerSeq = index + 1; }); + }, + () => { + if (state.committedEvents.length >= 64) state.committedEvents.splice(0, 1); + state.committedEvents.push(event(++sourceSeq, pick(eventTypes))); + state.ackedSourceSeq = sourceSeq; + }, + () => { + const index = Math.floor(next() * state.committedEvents.length); + const replay = state.committedEvents[index]; + if (!replay) return; + state = { ...state, committedEvents: state.committedEvents.map((candidate, position) => + position === index ? { ...replay, deliveryCount: replay.deliveryCount + 1 } : candidate), + replayDeliveries: state.replayDeliveries + 1 }; + }, + () => { state = structuredClone(state); }, + () => { state = { ...state, commands: [...state.commands], committedEvents: [...state.committedEvents] }; }, + () => { + state.connectionCount += 1; + const target = pick(state.commands); + if (target) state.commandDeliveryCounts[target.commandId] = (state.commandDeliveryCounts[target.commandId] ?? 0) + 1; + }, + () => { + // Same values, different key order: JSON output changes. + const target = pick(state.commands); + if (!target) return; + const { payload } = target; + delete (target as Partial).payload; + target.payload = payload; + }, + () => { + const target = pick(state.commands); + if (target) target.status = target.status; // eslint-disable-line no-self-assign + }, + () => { state.malformedFrames += 1; }, + ]; + const cache = durableRecoveryInternals.createCheckpointRecordCache(); + for (let step = 0; step < 600; step += 1) { + pick(operations)!(); + const cached = durableRecoveryInternals.captureCheckpoint(state, runnerVersion, runnerDigest, cache); + const fresh = durableRecoveryInternals.captureCheckpoint(state, runnerVersion, runnerDigest); + expect(entries(cached), `seed ${seed}, step ${step}`).toEqual(entries(fresh)); + } + expect(state.commands.length + state.committedEvents.length).toBeGreaterThan(0); +}); diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-checkpoint.benchmark.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-checkpoint.benchmark.test.ts new file mode 100644 index 00000000000..056f8e182e6 --- /dev/null +++ b/packages/paperclip-runner/src/control-plane/durable-prp-checkpoint.benchmark.test.ts @@ -0,0 +1,84 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; + +import { expect, it, vi } from "vitest"; + +import { DurablePrpControlPlane, type DurablePrpControlPlaneStore } from "./durable-prp-control-plane.js"; + +// Opt in explicitly. Run from packages/paperclip-runner: +// PAPERCLIP_CHECKPOINT_FLUSH_BENCHMARK=1 pnpm exec vitest run src/control-plane/durable-prp-checkpoint.benchmark.test.ts +// Each case commits one new event and flushes the journal checkpoint with a +// no-op persistence callback, so it measures only the controller's own work. +// CPU time shares the host's background load; the serialised byte and call +// counts are exact and do not depend on load. +const enabled = process.env.PAPERCLIP_CHECKPOINT_FLUSH_BENCHMARK === "1"; +const flushes = Number(process.env.PAPERCLIP_CHECKPOINT_FLUSH_COUNT ?? "200"); + +type State = DurablePrpControlPlaneStore["state"]; +type WorkingStore = DurablePrpControlPlaneStore & { commit(candidate: State, checkpointRequired?: boolean): void }; + +const identity = { + runnerInstanceId: "runner-flush-benchmark", environmentLeaseId: "environment-flush-benchmark", + runId: "00000000-0000-4000-8000-000000000201", normalizedSessionId: "session-flush-benchmark", + turnId: "turn-flush-benchmark", itemId: "item-flush-benchmark", +}; + +function command(controllerSeq: number): State["commands"][number] { + return { schema: "paperclip.prp.command.v1", commandId: `command-${controllerSeq}`, controllerSeq, type: "session.snapshot", + issuedAt: "2026-11-13T00:00:00.000Z", payload: { text: "c".repeat(512) }, status: "completed", result: { ok: true } }; +} + +function event(sourceSeq: number): State["committedEvents"][number] { + return { sourceSeq, sourceEventId: `event-${sourceSeq}`, eventType: "item.completed", priority: 1, + envelope: { kind: "event", payload: { sourceSeq, eventType: "item.completed", payload: { text: "e".repeat(1024) } } }, + deliveryCount: 1, logicalEffectCount: 1 }; +} + +it.skipIf(!enabled).each([ + { name: "10 commands", commands: 10, events: 10 }, + { name: "500 commands", commands: 500, events: 10 }, + { name: "4,096 events", commands: 10, events: 4_096 }, +])("measures one checkpoint flush with $name", async ({ name, commands, events }) => { + const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-flush-benchmark-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, + expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + try { + const working = core.store as WorkingStore; + working.commit({ ...working.state, ackedSourceSeq: events, + commands: Array.from({ length: commands }, (_, index) => command(index + 1)), + committedEvents: Array.from({ length: events }, (_, index) => event(index + 1)) }); + let persisted = 0; + core.setCheckpointPersistence(async () => { persisted += 1; }); + await core.flushCheckpoint(); + const stringify = JSON.stringify; + let serialisedBytes = 0, serialisedRecords = 0; + const spy = vi.spyOn(JSON, "stringify").mockImplementation(((...args: Parameters) => { + const output = stringify(...args); + const value = args[0]; + if (value !== null && typeof value === "object" && ("controllerSeq" in value || "sourceSeq" in value)) serialisedRecords += 1; + serialisedBytes += output?.length ?? 0; + return output; + }) as typeof JSON.stringify); + let cpu = 0n; + for (let index = 0; index < flushes; index += 1) { + const sourceSeq = events + index + 1; + const state = working.state; + // One committed event, with the window kept at its configured length. + working.commit({ ...state, ackedSourceSeq: sourceSeq, committedEvents: [...state.committedEvents.slice(1), event(sourceSeq)] }); + const started = process.cpuUsage(); + await core.flushCheckpoint(); + const used = process.cpuUsage(started); + cpu += BigInt(used.user + used.system); + } + spy.mockRestore(); + expect(persisted).toBe(flushes + 1); + console.info(JSON.stringify({ case: name, flushes, + cpuMicrosecondsPerFlush: Number(cpu / BigInt(flushes)), + serialisedBytesPerFlush: Math.round(serialisedBytes / flushes), + serialisedRecordsPerFlush: serialisedRecords / flushes })); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}, 300_000); diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 3b1d71514ba..f49a1c00d75 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -402,23 +402,72 @@ function evictionCandidate(state: StoredCoreState): number { function checkpointChanges(records: ReadonlyMap, runId: string, previous?: DurablePrpCheckpoint) { const reset = !previous || previous.runId !== runId; - return { - changedRecords: new Map([...records].filter(([key, value]) => reset || previous.records.get(key) !== value)), - removedRecords: reset ? [] : [...previous.records.keys()].filter(key => !records.has(key)), - reset, - }; + if (reset) return { changedRecords: new Map(records), removedRecords: [], reset }; + // Unchanged records keep the same string, so most comparisons are by reference. + const changedRecords = new Map(); + for (const [key, value] of records) if (previous.records.get(key) !== value) changedRecords.set(key, value); + const removedRecords: string[] = []; + for (const key of previous.records.keys()) if (!records.has(key)) removedRecords.push(key); + return { changedRecords, removedRecords, reset }; } -function captureCheckpoint(state: StoredCoreState, runnerVersion: string, runnerDigest: string): DurablePrpCheckpoint { +/** + * Reuses the JSON of a command or event record while the record is unchanged, + * so a checkpoint serialises only what changed since the last one, not the + * whole journal. Commands and events are flat: their fields are edited in place + * (status, result, controllerSeq), but their nested payload, result and + * envelope objects are owned copies that are replaced, never edited. The JSON + * is therefore reused only while the record has the same keys, in the same + * order, with identical values. + */ +class CheckpointRecordCache { + #entries = new WeakMap(); + + /** The record's part name and JSON. The name derives from a field, so it is reused with the JSON. */ + record(prefix: "command:" | "event:", sequence: number, value: object): CheckpointRecord { + const entry = this.#entries.get(value); + if (entry && sameRecordFields(value, entry.fields)) return entry; + const fields: unknown[] = []; + for (const key in value) fields.push(key, (value as Record)[key]); + const created = { key: `${prefix}${sequence}`, json: JSON.stringify(value), fields }; + this.#entries.set(value, created); + return created; + } +} + +type CheckpointRecord = { readonly key: string; readonly json: string; readonly fields: readonly unknown[] }; + +function sameRecordFields(value: object, fields: readonly unknown[]): boolean { + let index = 0; + for (const key in value) { + if (fields[index] !== key || fields[index + 1] !== (value as Record)[key]) return false; + index += 2; + } + return index === fields.length; +} + +function uncachedRecord(prefix: "command:" | "event:", sequence: number, value: object): Omit { + return { key: `${prefix}${sequence}`, json: JSON.stringify(value) }; +} + +function captureCheckpoint(state: StoredCoreState, runnerVersion: string, runnerDigest: string, + cache?: CheckpointRecordCache): DurablePrpCheckpoint { const runId = checkpointRunId(state); const records = new Map(); + const record = cache ? cache.record.bind(cache) : uncachedRecord; const { commands, committedEvents, ackedSourceSeq: _cursor, replayDeliveries: _replays, stagedRunAttachment, ...metadata } = state; const stagedDescriptor = stagedRunAttachment ? (({ runAttachTemplate: _template, ...descriptor }) => descriptor)(stagedRunAttachment) : undefined; if (stagedRunAttachment?.commandId === null) records.set("staged-attachment-template", JSON.stringify(stagedRunAttachment.runAttachTemplate)); const latestUsage = committedEvents.findLast(event => event.eventType === "usage.reported"); const recoveryEvents = committedEvents.filter(event => !isStreamingEvent(event) || event === latestUsage); - for (const command of commands) records.set(`command:${command.controllerSeq}`, JSON.stringify(command)); - for (const event of recoveryEvents) records.set(`event:${event.sourceSeq}`, JSON.stringify(event)); + for (const command of commands) { + const { key, json } = record("command:", command.controllerSeq, command); + records.set(key, json); + } + for (const event of recoveryEvents) { + const { key, json } = record("event:", event.sourceSeq, event); + records.set(key, json); + } const header: CheckpointHeader = { schema: "paperclip.runner.controller-checkpoint.v2", runnerVersion, runnerDigest, state: { ...metadata, ...(stagedDescriptor ? { stagedRunAttachment: stagedDescriptor } : {}), checkpointEventPolicy: "control-v1" } }; records.set("header", JSON.stringify(header)); @@ -426,7 +475,9 @@ function captureCheckpoint(state: StoredCoreState, runnerVersion: string, runner commandCount: commands.length, eventCount: recoveryEvents.length })); const maintenance = new Set(["turn.stop", "runner.drain", "runner.suspend", "session.snapshot", "session.close"]); return { - runnerVersion, runnerDigest, runId, records, ...checkpointChanges(records, runId), + // The controller replaces this full write set with a delta before it persists. + runnerVersion, runnerDigest, runId, records, + ...(cache ? { changedRecords: records, removedRecords: [], reset: true } : checkpointChanges(records, runId)), recoveryEvidence: { runnerInstanceId: state.identity.runnerInstanceId, normalizedSessionId: state.identity.normalizedSessionId, authenticatedConnectionCount: state.connectionCount, highestRecordedSourceSeq: state.committedEvents.reduce((highest, event) => Math.max(highest, event.sourceSeq), state.ackedSourceSeq) }, @@ -792,7 +843,12 @@ function canonicalJson( } } -export const durableRecoveryInternals = Object.freeze({ canonicalJson }); +export const durableRecoveryInternals = Object.freeze({ + canonicalJson, + captureCheckpoint: (state: DurablePrpControlPlaneStore["state"], runnerVersion: string, runnerDigest: string, + cache?: CheckpointRecordCache) => captureCheckpoint(state, runnerVersion, runnerDigest, cache), + createCheckpointRecordCache: () => new CheckpointRecordCache(), +}); function canonicalDigest(value: unknown): string { return createHash("sha256").update(canonicalJson(value)).digest("hex"); @@ -2138,6 +2194,7 @@ export class DurablePrpControlPlane { #persistedRevision = -1; #queuedRevision = -1; #lastCheckpoint?: DurablePrpCheckpoint; + readonly #checkpointRecords = new CheckpointRecordCache(); #publishedState?: StoredCoreState; #publishedStore?: DurablePrpControlPlaneStore; #publicationCommands = new Map(); @@ -2296,7 +2353,7 @@ export class DurablePrpControlPlane { this.#store.useExternalPersistence(); this.#persistCheckpoint = persist; this.#publishedState = this.#capturePublishedState(captureCheckpoint(this.#store.state, - this.#expectedRunnerVersion, this.#expectedRunnerDigest)); + this.#expectedRunnerVersion, this.#expectedRunnerDigest, this.#checkpointRecords)); const core = this; this.#publishedStore = { path: this.#store.path, get state() { return core.#visibleState(); } }; } @@ -2307,7 +2364,7 @@ export class DurablePrpControlPlane { return this.#persistenceTail; } const snapshot = captureCheckpoint(this.#store.state, - this.#expectedRunnerVersion, this.#expectedRunnerDigest); + this.#expectedRunnerVersion, this.#expectedRunnerDigest, this.#checkpointRecords); const published = this.#capturePublishedState(snapshot); const revision = this.#store.revision; this.#queuedRevision = revision; @@ -3031,7 +3088,9 @@ export class DurablePrpControlPlane { controllerSeq, type, issuedAt: new Date().toISOString(), - payload, + // The journal owns its copy: checkpoints reuse a command's JSON while + // its fields keep the same values, so a caller must not edit it later. + payload: structuredClone(payload), status: "pending", result: null, };