diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index 918779e5261..bd4f062de21 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -787,6 +787,134 @@ it.each([false, true])("recovers checkpoint persistence and the same connection } }); +it("keeps one waiting checkpoint behind a slow write and saves every queued command before resolving", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-coalesce-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + const stored = new Map(); + const writes: DurablePrpCheckpoint[] = []; + let release!: () => void; + let gate: Promise = Promise.resolve(); + core.setCheckpointPersistence(async snapshot => { + writes.push(snapshot); + await gate; + if (snapshot.reset) stored.clear(); + for (const key of snapshot.removedRecords) stored.delete(key); + for (const [key, value] of snapshot.changedRecords) stored.set(key, value); + }); + try { + await core.flushCheckpoint(); + gate = new Promise(resolveWrite => { release = resolveWrite; }); + core.queueCommand("session.snapshot", {}, "in-flight-command"); + const inFlight = core.flushCheckpoint(); + await vi.waitFor(() => expect(writes).toHaveLength(2)); + const settled: string[] = []; + const waiting = Array.from({ length: 40 }, (_, index) => { + const commandId = `waiting-command-${index}`; + core.queueCommand("session.snapshot", {}, commandId); + return core.flushCheckpoint().then(() => { + expect([...stored.values()].join("\n")).toContain(commandId); + settled.push(commandId); + }); + }); + await new Promise(resolveTurn => setImmediate(resolveTurn)); + expect(writes).toHaveLength(2); + expect(settled).toEqual([]); + release(); + await inFlight; + await Promise.all(waiting); + expect(writes).toHaveLength(3); + expect(settled).toHaveLength(40); + expect(stored).toEqual(writes.at(-1)!.records); + expect(decodeDurablePrpCheckpoint(stored).records).toEqual(writes.at(-1)!.records); + await core.flushCheckpoint(); + expect(writes).toHaveLength(3); + } finally { + release?.(); + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it("writes a waiting checkpoint for each run in order when the run changes behind a slow write", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-coalesce-handoff-")); + const target = { ...identity, runId: "00000000-0000-4000-8000-000000000002", turnId: "next-turn", itemId: "next-item" }; + const template = { provider: { provider: "codex", command: "fixture" }, authorizedTools: {} }; + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + const writes: DurablePrpCheckpoint[] = []; + let release!: () => void; + let gate: Promise = Promise.resolve(); + core.setCheckpointPersistence(async snapshot => { writes.push(snapshot); await gate; }); + const contents = (checkpoint: DurablePrpCheckpoint) => [...checkpoint.records.values()].join("\n"); + try { + await core.flushCheckpoint(); + gate = new Promise(resolveWrite => { release = resolveWrite; }); + core.queueCommand("session.snapshot", {}, "source-in-flight"); + const inFlight = core.flushCheckpoint(); + await vi.waitFor(() => expect(writes).toHaveLength(2)); + core.queueCommand("session.snapshot", {}, "source-waiting"); + const sourceWaiting = core.flushCheckpoint(); + core.stageRunAttachment(target, template); + const targetWaiting = core.flushCheckpoint(); + core.queueCommand("session.snapshot", {}, "target-later"); + const targetLater = core.flushCheckpoint(); + expect(targetLater).toBe(targetWaiting); + expect(sourceWaiting).not.toBe(targetWaiting); + release(); + await Promise.all([inFlight, sourceWaiting, targetWaiting, targetLater]); + expect(writes.map(write => write.runId)).toEqual([identity.runId, identity.runId, identity.runId, target.runId]); + expect(contents(writes[2]!)).toContain("source-waiting"); + expect(contents(writes[2]!)).not.toContain("target-later"); + expect(writes[2]!.records.has("staged-attachment-template")).toBe(false); + expect(writes[3]).toMatchObject({ reset: true, predecessor: { runId: identity.runId } }); + expect(contents(writes[3]!)).toContain("target-later"); + expect(writes[3]!.records.has("staged-attachment-template")).toBe(true); + await core.flushCheckpoint(); + expect(writes).toHaveLength(4); + } finally { + release?.(); + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + +it("rejects every caller waiting on a coalesced checkpoint whose write fails", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-coalesce-failure-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + const writes: DurablePrpCheckpoint[] = []; + let release!: () => void; + let gate: Promise = Promise.resolve(); + let fail = true; + core.setCheckpointPersistence(async snapshot => { + writes.push(snapshot); + await gate; + if (fail && [...snapshot.records.values()].join("\n").includes("second-waiting")) throw new Error("coalesced write failed"); + }); + try { + await core.flushCheckpoint(); + gate = new Promise(resolveWrite => { release = resolveWrite; }); + core.queueCommand("session.snapshot", {}, "in-flight-command"); + const inFlight = core.flushCheckpoint(); + await vi.waitFor(() => expect(writes).toHaveLength(2)); + const waiting = ["first-waiting", "second-waiting"].map(commandId => { + core.queueCommand("session.snapshot", {}, commandId); + return core.flushCheckpoint(); + }); + const rejected = waiting.map(promise => expect(promise).rejects.toThrow("coalesced write failed")); + release(); + await inFlight; + await Promise.all(rejected); + expect(writes).toHaveLength(3); + fail = false; + await core.flushCheckpoint(); + expect(writes.at(-1)!.reset).toBe(true); + expect([...writes.at(-1)!.records.values()].join("\n")).toContain("second-waiting"); + } finally { + release?.(); + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); + it("rebuilds a queued checkpoint after the preceding write fails", async () => { const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-queued-retry-")); const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 6f73fe23383..8e75bf1ff8c 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -2131,6 +2131,7 @@ export class DurablePrpControlPlane { #visibleStateCache?: { working: StoredCoreState; published: StoredCoreState; revision: number; state: StoredCoreState }; #checkpointPersistenceFailed = false; #persistenceTail = Promise.resolve(); + #waitingCheckpoint: { snapshot: DurablePrpCheckpoint; published: StoredCoreState; revision: number } | null = null; #sendQueues = new Map>(); #recoveredCursor: { runId: string; sourceSeq: number } | null = null; #readableStateListeners = new Set<() => void>(); @@ -2297,8 +2298,19 @@ export class DurablePrpControlPlane { const published = this.#capturePublishedState(snapshot); const revision = this.#store.revision; this.#queuedRevision = revision; + const waiting = this.#waitingCheckpoint; + if (waiting !== null && waiting.snapshot.runId === snapshot.runId) { + waiting.snapshot = snapshot; + waiting.published = published; + waiting.revision = revision; + return this.#persistenceTail; + } + const job = { snapshot, published, revision }; + this.#waitingCheckpoint = job; const persist = this.#persistCheckpoint; const pending = this.#persistenceTail.catch(() => undefined).then(async () => { + if (this.#waitingCheckpoint === job) this.#waitingCheckpoint = null; + const { snapshot, published, revision } = job; // The previous write may have failed after committing. Rebuild the delta // at execution time, and reset after uncertainty instead of assuming it failed. const checkpoint = { ...snapshot, ...checkpointChanges(snapshot.records, snapshot.runId, diff --git a/scripts/benchmark-runner-checkpoint-backlog.ts b/scripts/benchmark-runner-checkpoint-backlog.ts new file mode 100644 index 00000000000..3323a7fb089 --- /dev/null +++ b/scripts/benchmark-runner-checkpoint-backlog.ts @@ -0,0 +1,120 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import { DurablePrpControlPlane } from "../packages/paperclip-runner/src/control-plane/durable-prp-control-plane.js"; + +const numberOption = (name: string, fallback: number) => { + const value = process.argv.find(arg => arg.startsWith(`--${name}=`))?.slice(name.length + 3); + return value === undefined ? fallback : Number(value); +}; +const runs = numberOption("runs", 8); +const seconds = numberOption("seconds", 30); +const eventsPerSecond = numberOption("events-per-second", 13); +const commandsPerSecond = numberOption("commands-per-second", 2); +const writeMs = numberOption("write-ms", 1000); +const eventBytes = numberOption("event-bytes", 2048); +const seedEvents = numberOption("seed-events", 300); + +setFlagsFromString("--expose-gc"); +const gc = runInNewContext("gc") as () => void; +const pause = (ms: number) => new Promise(resolveTimer => setTimeout(resolveTimer, ms)); +const settledHeap = async () => { + for (let index = 0; index < 6; index++) { + gc(); + await new Promise(resolveTurn => setImmediate(resolveTurn)); + } + return process.memoryUsage().heapUsed; +}; + +type WorkingStore = { + revision: number; + state: DurablePrpControlPlane["store"]["state"]; + commit(candidate: DurablePrpControlPlane["store"]["state"], checkpointRequired?: boolean): void; +}; +const identity = (index: number) => ({ + runnerInstanceId: `runner-bench-${index}`, environmentLeaseId: `environment-bench-${index}`, + runId: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`, + normalizedSessionId: `session-bench-${index}`, turnId: `turn-bench-${index}`, itemId: `item-bench-${index}`, +}); + +const collected = { count: 0 }; +const registry = new FinalizationRegistry(() => { collected.count += 1; }); +const roots: string[] = []; +const baseline = await settledHeap(); +let writesStarted = 0, revisionsRequested = 0; + +function benchEvent(index: number, sourceSeq: number) { + return { sourceSeq, sourceEventId: `bench-${index}-${sourceSeq}`, eventType: "item.completed", priority: 1 as const, + envelope: { kind: "event", payload: { sourceSeq, eventType: "item.completed", payload: { text: "x".repeat(eventBytes) } } }, + deliveryCount: 1, logicalEffectCount: 1 as const }; +} + +async function startRun(index: number) { + const root = mkdtempSync(resolve(tmpdir(), "runner-checkpoint-backlog-")); + roots.push(root); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity: identity(index), + expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + const working = core.store as unknown as WorkingStore; + const seeded = Array.from({ length: seedEvents }, (_, offset) => benchEvent(index, offset + 1)); + working.commit({ ...working.state, ackedSourceSeq: seedEvents, committedEvents: seeded }); + let lastRevision = -1; + core.setCheckpointPersistence(async () => { + writesStarted += 1; + await pause(writeMs); + }); + const flush = core.flushCheckpoint.bind(core); + core.flushCheckpoint = () => { + if (working.revision !== lastRevision) { lastRevision = working.revision; revisionsRequested += 1; } + return flush(); + }; + registry.register(core, index); + return { core, working }; +} + +async function driveEvents(run: Awaited>, until: number, index: number) { + let sourceSeq = run.working.state.ackedSourceSeq; + while (Date.now() < until) { + sourceSeq += 1; + const state = run.working.state; + run.working.commit({ ...state, ackedSourceSeq: sourceSeq, + committedEvents: [...state.committedEvents, benchEvent(index, sourceSeq)].slice(-4096) }); + await Promise.all([run.core.flushCheckpoint(), pause(1000 / eventsPerSecond)]); + } +} + +async function driveCommands(run: Awaited>, until: number, index: number) { + const pending: Promise[] = []; + for (let count = 0; commandsPerSecond > 0 && Date.now() < until && count < 400; count++) { + run.core.queueCommand("session.snapshot", {}, `bench-command-${index}-${count}`); + pending.push(run.core.flushCheckpoint()); + await pause(1000 / commandsPerSecond); + } + await Promise.all(pending); +} + +let active = await Promise.all(Array.from({ length: runs }, (_, index) => startRun(index))); +const until = Date.now() + seconds * 1000; +const driving = Promise.all(active.flatMap((run, index) => [driveEvents(run, until, index), driveCommands(run, until, index)])); +await pause(seconds * 500); +const midRun = await settledHeap(); +await driving; +await Promise.all(active.map(run => run.core.flushCheckpoint())); +const live = await settledHeap(); +const committedEvents = active[0]!.working.state.committedEvents.length; +await Promise.all(active.map(run => run.core.stop())); +active = []; +for (let attempt = 0; attempt < 40 && collected.count < runs; attempt++) { + await settledHeap(); + await pause(25); +} +const finished = await settledHeap(); +for (const root of roots) rmSync(root, { recursive: true, force: true }); +const kilobytes = (bytes: number) => Math.round(bytes / runs / 1024); +console.log(JSON.stringify({ + settings: { runs, seconds, eventsPerSecond, commandsPerSecond, writeMs, eventBytes, seedEvents }, + newEventsCommittedPerRun: committedEvents - seedEvents, revisionsFlushed: revisionsRequested, checkpointWrites: writesStarted, + midRunRetainedKBPerRun: kilobytes(midRun - baseline), idleLiveRetainedKBPerRun: kilobytes(live - baseline), + finishedRetainedKBPerRun: kilobytes(finished - baseline), controlPlanesCollected: `${collected.count}/${runs}`, +}));