From 0bba04675c42bae79ecd6d67d895e8ef902d0c56 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:59:01 -0600 Subject: [PATCH] perf(run-events): append in-order native events in one statement (#21511) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - One server process stores every event of every running native agent in `heartbeat_run_events` > - After #21460, text deltas commit in batches, but each batch still takes five statements: BEGIN, the run lock, a replay probe, one insert, and COMMIT. A single event without a wake takes four > - In the agent scaling load model, the process issues about 36 statements per agent per second and saturates its one thread between 200 and 400 agents > - This pull request writes a batch of in-order native events, or one such event, with one statement that commits on its own > - The benefit is about half the statements per agent, less CPU per agent on the main thread, and less time on a pool connection, with the same rows, receipts, cursor and locks ## Linked Issues or Issue Description Refs #21460 (batched deltas). Refs #21435 (source cursor). Refs #21418 (agent scaling plan and load model). **What existing behavior does this improve?** The append of native source events that continue their source's stored cursor: batched text deltas, the first delta after another event, and every other native event that registers no wake. **Subsystem affected** Server run event ingest: `server/src/services/heartbeat-run-events.ts`, and the port comment in `paperclip-control-plane-port.ts`. **Current behavior** A delta batch takes BEGIN, a run lock read (`for no key update`), a replay probe, one insert, and COMMIT: five statements. One delta that commits at once takes four. In the load model at 200 agents the process sends about 6,700 statements per second at event-loop utilization 0.72. **Proposed behavior** The same append is one statement. Its update of the run row takes the same row lock. It requires the stored cursor to end just before the batch, requires the `next_event_seq` that its own snapshot read, and finds no replayed event. Then it moves the sequence and the cursor and inserts every row. If any check fails, it writes nothing and the append takes the existing locked transaction. **Reason and benefit** Fewer statements free the main thread and the shared pool for other agents. In the load model at 200 agents, statements per second fall from about 6,700 to about 3,300 and event-loop utilization falls from 0.72 to 0.56. **Breaking changes** None. No schema change. Rows, receipts and the cursor are the same. Servers from before this change keep working on the same runs. ## What Changed - `appendNewNativeSourceEvents` (`heartbeat-run-events.ts`) is one statement: a CTE reads `next_event_seq` from the statement snapshot, an `update heartbeat_runs ... returning` locks the run row and moves `next_event_seq` and the source cursor, and an `insert ... select from jsonb_array_elements($9)` writes every row. The text is fixed, with nine bind parameters for any batch size. - **Why the `next_event_seq` check.** A statement that waits for the run row lock keeps the snapshot it took before the wait. Its replay probe cannot see events that the lock holder committed. Every event append (native, ordinary, the run coordinator store, the retry queue) moves `next_event_seq` in its own transaction. So the update requires the value that its snapshot read. PostgreSQL re-checks that condition on the newest row version after the wait, and any append committed meanwhile makes the statement write nothing. The unique indexes on `(run_id, source_event_id)` and `(run_id, source_instance_id, source_seq)` are a second guard: a unique violation from this statement also falls back. - A source without a cursor starts at 0. The probe still finds any stored event of that source, so a run that an older server wrote falls back and recounts. - `appendHeartbeatRunEventBatch` and `appendHeartbeatRunEvent` try the one statement first for native events without a wake, outside a caller's transaction. Every other case (a replay, a gap, an event with a wake, a cursor that lags, a run changed meanwhile, a binding mismatch, a run deleted meanwhile, a caller's transaction) takes the existing transaction, unchanged. - Inside a caller's transaction the append keeps its SAVEPOINT path. A failed statement there would abort the caller's transaction, and the fallback could not run. - `assertWritable` runs before the statement is sent and after it returns (see Risks). ## Verification - Red/green on embedded PostgreSQL. I ran the new tests against the previous `heartbeat-run-events.ts` and saw 9 of them fail; all 25 tests in the file pass with the change (`server/src/__tests__/heartbeat-run-event-atomic-append.test.ts`): - One statement for an in-order event and for a batch (before: 4 and 5). - A writer revoked before the send sends nothing; a statement that fails on its third row stores no row and no sequence; the resend commits. - A writer revoked while the statement runs: the rows stay, the receipt is rejected, and the resend is a replay with the same rows. - A batch waits for Stop on the run row and appends after it; a batch and a single event that wait for a run deletion fail with `heartbeat_run_event_binding_mismatch` and store nothing. - An older server stores events 2 and 3 while it holds the run row, without the cursor: the waiting batch 2–4 falls back, returns duplicate, duplicate, committed, and the cursor ends at 4. - Two connections append the same 200-event stream in batches of 10: each event is stored once, in source order, sequences 1–200 without a gap, one `committed` and one `duplicate` receipt per event. - A 7,000-event batch (more than any bind parameter limit at 10 parameters per row) is one statement. Before: it fell back to one statement per event and timed out at 15 s. - A run without a cursor starts at its first event; a run with events from an older server replays and then continues. - Equivalence: 12 random streams of 30 steps (in-order batches, gaps, replays of earlier ranges, fills of gaps, ordinary events, conflicting replays). Each step goes to two runs, one through the new path and one forced through the one-by-one locked path. Receipts, errors, rows, `next_event_seq` and cursors are equal at every step. - Mutation check: with the `next_event_seq` check and the unique-violation fallback removed, the older-server race test fails. - The port suite (`paperclip-control-plane-port.test.ts`, with the package conformance suite) passes: 125 tests. Two tests changed: the streamed-event test now expects one statement and the new revocation behavior, and the batch test checks that the five deltas are one statement sent before the boundary event. - Other focused suites pass, one file at a time: `heartbeat-run-event-sequencing` (5), `native-event-boundary-golden` (1), `native-observer-replay` (1), `paperclip-semantic-conformance` (1), `native-interaction-bridge` (7), `chat-native-progress-work` (19), `heartbeat-native-restart-suspension` (115), `live-recovery-work` (51). In 2 of 3 runs with this change, all 51 tests passed but the file's `afterAll` (closing the side pools and the embedded database) hit its 30 s timeout on this busy machine; the third run passed cleanly, and one run on `origin/master` passed. The native events in that file all register a wake, so they do not take the new path. - `pnpm --filter @paperclipai/server run typecheck` and `pnpm check:module-boundaries` pass. - Load model: `scripts/benchmark-agent-scaling-load.ts` as merged in #21418 (per-source counts stop when the window closes). I did not change its batching rules for this pull request. Model rates: 20 deltas/s, 1 other event/s, 0.3 API requests/s per agent; default pool of 10 connections; embedded PostgreSQL on the same, shared machine. I ran "before" (`origin/master`) and "after" one after the other, with a pause between runs, and I give the load average at the start of each run. The machine was busy with other work (load 10–15 on 18 cores), so compare rows that ran close together. No run had a failed operation, a failed checkpoint save or a lost lease. | Agents | Code | Window | Load avg | Event-loop utilization | Statements/s | Delta lateness p50 / p99 (ms) | Other events done / offered | Other event lateness p50 / p99 (ms) | | --- | --- | --- | --- | --- | --- | --- | --- | --- | | 200 | before | 60 s | 9.6 | 0.56 | 7,043 | 283 / 3,368 | 12,000 / 12,000 | 278 / 3,895 | | 400 | before | 60 s | 11.8 | 0.59 | 7,158 | 9,927 / 18,779 | 6,068 / 24,000 | 1,904 / 60,154 | | 400 | after | 60 s | 12.7 | 0.74 | 5,465 | 1,129 / 6,040 | 18,719 / 24,000 | 5,721 / 15,238 | | 600 | before | 60 s | 15.0 | 0.85 | 12,419 | 7,108 / 11,017 | 7,836 / 36,000 | 10,822 / 52,171 | | 600 | after | 60 s | 10.0 | 0.69 | 6,158 | 2,357 / 9,483 | 18,771 / 36,000 | 2,280 / 30,985 | | 600 | after, `DELTA_BATCH_MS=0` | 60 s | 10.8 | 0.68 | 8,014 | 36,135 / 52,222 | 36,000 / 36,000 | 698 / 3,422 | At 200 agents an earlier pair gave 0.61 → 0.43 event-loop utilization and 7,051 → 3,210 statements/s. One process with this change alone still does not carry 600 agents: at 400 and above the 10-connection pool and the shared database are the limit (event-loop utilization stays below 0.75 while other events fall behind). #21520 and a third pull request for the remaining statements of other events and checkpoint saves continue from here; their description has the stacked results. - Statements: a delta batch 5 → 1; one delta or one native event without a wake 4 → 1. Unchanged: an event with a wake (6) and a checkpoint save (5). ## Risks - **`assertWritable` window (read first).** Before, the writer check ran inside the transaction, before the first statement and before COMMIT. A revocation while the statements ran rolled the append back. Now a revocation while the one statement runs does not roll it back: the rows commit, and the caller gets the error instead of the receipt, so no observer runs and the runner gets no ack. This check guards a controller that hands its run to a successor at restart (`nativeRunsDetachingForRestart`). I think this is acceptable: (1) the earlier check was not atomic with COMMIT either, so a revocation during the COMMIT round trip already committed; the window grows from one COMMIT round trip to one statement round trip plus its wait for a pool connection; (2) the append is idempotent by source event id and payload hash, so the successor sees the committed row as a replay of the event that the runner resends; (3) the old controller never acts on the rejected receipt. A new test covers exactly this. - A crash loses nothing new: the statement commits or not as a whole, and a resend after an unknown outcome is a replay. - The check on `next_event_seq` makes the statement write nothing whenever any other event of the run committed after its snapshot. Under heavy contention on one run, more appends take the fallback. The fallback is the previous code path. - The statement text is fixed SQL sent through the driver (`$client.unsafe`), like the rest of the event stream statements. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../heartbeat-run-event-atomic-append.test.ts | 247 ++++++++++++++++-- server/src/services/heartbeat-run-events.ts | 106 +++++++- .../paperclip-control-plane-port.test.ts | 34 +-- .../paperclip-control-plane-port.ts | 2 - 4 files changed, 334 insertions(+), 55 deletions(-) diff --git a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts index e13cb4fe461..e7da91ca4b7 100644 --- a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts +++ b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts @@ -130,26 +130,24 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { protocolSchemaVersion: 1, canonicalPayload: { sourceSeq } }, }); - it("appends a native source event with one lock read and one write statement", async () => { + it("appends an in-order native source event in one statement, and a replay under the run lock", async () => { const runId = await makeRun(); const first = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1))); - // BEGIN, the run lock, the append, COMMIT. The lock is a statement of its - // own so that the append reads a snapshot taken after the lock was granted. - expect(first.statements).toHaveLength(4); - expect(first.statements[0]).toMatch(/^begin/i); - expect(first.statements[1]).toMatch(/^select "company_id", "agent_id", \("native_source_cursors" ->> \$1\)::bigint as "source_cursor" from "heartbeat_runs" .* for no key update$/); - expect(first.statements[2]).toMatch(/^with existing as .* insert into heartbeat_run_events /); - expect(first.statements[3]).toMatch(/^commit/i); + expect(first.statements).toHaveLength(1); + expect(first.statements[0]).toMatch(/^with snapshot_run as .* update heartbeat_runs .* insert into heartbeat_run_events /); expect(first.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 1 }); expect(first.result.row.createdAt).toBeInstanceOf(Date); expect(await readEvents(runId)).toEqual([first.result.row]); const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1))); - expect(replay.statements).toHaveLength(4); + expect(replay.statements).toHaveLength(5); + expect(replay.statements[1]).toMatch(/^begin/i); + expect(replay.statements[2]).toMatch(/^select "company_id", "agent_id", \("native_source_cursors" ->> \$1\)::bigint as "source_cursor" from "heartbeat_runs" .* for no key update$/); + expect(replay.statements[3]).toMatch(/^with existing as .* insert into heartbeat_run_events /); + expect(replay.statements[4]).toMatch(/^commit/i); expect(replay.result).toEqual({ row: first.result.row, disposition: "duplicate", highestContiguousSourceSeq: 1 }); expect((await readRun(runId)).nextEventSeq).toBe(2); - // A caller's transaction adds only its SAVEPOINT; a writer check adds nothing. const nested = await sentStatements(db.$client, () => db.transaction(tx => appendHeartbeatRunEvent(tx as unknown as typeof db, nativeEvent(runId, 2)))); expect(nested.statements).toHaveLength(5); @@ -157,7 +155,7 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { const checked = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, { ...nativeEvent(runId, 3), assertWritable: () => { checks += 1; }, })); - expect(checked.statements).toHaveLength(4); + expect(checked.statements).toHaveLength(1); expect(checks).toBe(2); expect(checked.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 3 }); expect((await readEvents(runId)).map(row => [row.seq, row.sourceSeq])).toEqual([[1, 1], [2, 2], [3, 3]]); @@ -169,13 +167,13 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { for (let sourceSeq = 2; sourceSeq <= 6; sourceSeq += 1) { const appended = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, sourceSeq))); expect(appended.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: sourceSeq }); - expect(appended.statements).toHaveLength(4); + expect(appended.statements).toHaveLength(1); // The append statement reads no other event of the run. - expect(appended.statements[2]).not.toMatch(/count\(|max\(source_seq\)/); + expect(appended.statements[0]).not.toMatch(/count\(|max\(source_seq\)/); } const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 3))); expect(replay.result).toMatchObject({ disposition: "duplicate", highestContiguousSourceSeq: 6 }); - expect(replay.statements[2]).not.toMatch(/count\(/); + expect(replay.statements[3]).not.toMatch(/count\(/); expect((await readRun(runId)).nativeSourceCursors).toEqual({ "native-writer": 6 }); }); @@ -205,14 +203,13 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { expect((await readEvents(runId)).map(row => row.sourceSeq)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 10, 9]); }); - it("commits a batch of in-order deltas with one lock read, one replay probe and one insert", async () => { + it("commits a batch of in-order deltas in one statement", async () => { const runId = await makeRun(); await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); await appendHeartbeatRunEvent(db, { companyId, agentId, runId, eventType: "stdout", message: "ordinary" }); const batch = [2, 3, 4, 5, 6].map(sourceSeq => nativeEvent(runId, sourceSeq)); const appended = await sentStatements(db.$client, () => appendHeartbeatRunEventBatch(db, batch)); - // BEGIN, the run lock, the replay probe, one insert of every row, COMMIT. - expect(appended.statements).toHaveLength(5); + expect(appended.statements).toHaveLength(1); expect(appended.statements.filter(text => text.includes("insert into heartbeat_run_events"))).toHaveLength(1); expect(appended.result.map(receipt => [receipt.disposition, receipt.row.seq, receipt.row.sourceSeq, receipt.highestContiguousSourceSeq])) .toEqual([2, 3, 4, 5, 6].map((sourceSeq, index) => ["committed", index + 3, sourceSeq, sourceSeq])); @@ -254,10 +251,17 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); const before = await readRun(runId); const batch = [2, 3, 4].map(sourceSeq => nativeEvent(runId, sourceSeq)); - let checks = 0; - // The writer loses the run after the insert and before the commit. - await expect(appendHeartbeatRunEventBatch(db, batch, () => { if (++checks === 2) throw new Error("writer revoked"); })) - .rejects.toThrow("writer revoked"); + const revoked = await sentStatements(db.$client, () => appendHeartbeatRunEventBatch(db, batch, () => { + throw new Error("writer revoked"); + }).catch((error: Error) => error.message)); + expect(revoked).toEqual({ result: "writer revoked", statements: [] }); + await db.execute(sql`alter table heartbeat_run_events add constraint atomic_batch_failure + check (source_seq is distinct from 4) not valid`); + try { + await expect(appendHeartbeatRunEventBatch(db, batch)).rejects.toThrow(); + } finally { + await db.execute(sql`alter table heartbeat_run_events drop constraint atomic_batch_failure`); + } expect(await readRun(runId)).toEqual(before); expect((await readEvents(runId)).map(row => row.sourceSeq)).toEqual([1]); const resent = await appendHeartbeatRunEventBatch(db, batch); @@ -265,6 +269,203 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { .toEqual([["committed", 2, 2], ["committed", 3, 3], ["committed", 4, 4]]); }); + it("rejects the receipts of a writer revoked while its batch commits, and its resend is a replay", async () => { + const runId = await makeRun(); + await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); + const batch = [2, 3, 4].map(sourceSeq => nativeEvent(runId, sourceSeq)); + let checks = 0; + await expect(appendHeartbeatRunEventBatch(db, batch, () => { if (++checks === 2) throw new Error("writer revoked"); })) + .rejects.toThrow("writer revoked"); + expect((await readEvents(runId)).map(row => [row.seq, row.sourceSeq])).toEqual([[1, 1], [2, 2], [3, 3], [4, 4]]); + const resent = await appendHeartbeatRunEventBatch(db, batch); + expect(resent.map(receipt => [receipt.disposition, receipt.row.seq, receipt.highestContiguousSourceSeq])) + .toEqual([["duplicate", 2, 4], ["duplicate", 3, 4], ["duplicate", 4, 4]]); + expect(await readRun(runId)).toMatchObject({ nextEventSeq: 5, nativeSourceCursors: { "native-writer": 4 } }); + }); + + const holdRun = async (runId: string, work: (tx: typeof db) => Promise) => { + let release!: () => void, locked!: () => void; + const released = new Promise(resolve => { release = resolve; }); + const lockedPromise = new Promise(resolve => { locked = resolve; }); + const holder = otherDb.transaction(async tx => { + await tx.execute(sql`select id from heartbeat_runs where id = ${runId} for update`); + await work(tx as unknown as typeof db); + locked(); + await released; + }); + await lockedPromise; + return { release: () => { release(); return holder; } }; + }; + const settlesWhileHeld = async (pending: Promise) => { + let settled = false; + void pending.then(() => { settled = true; }, () => { settled = true; }); + await new Promise(resolve => setTimeout(resolve, 200)); + return settled; + }; + + it("waits for Stop on the run row and appends after it, and fails without rows after a run deletion", async () => { + const runId = await makeRun(); + await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); + const stop = await holdRun(runId, async tx => { + await tx.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, runId)); + }); + const batch = appendHeartbeatRunEventBatch(db, [2, 3].map(sourceSeq => nativeEvent(runId, sourceSeq))); + const batchSettledWhileHeld = await settlesWhileHeld(batch); + await stop.release(); + expect(batchSettledWhileHeld).toBe(false); + expect((await batch).map(receipt => [receipt.disposition, receipt.row.seq])).toEqual([["committed", 2], ["committed", 3]]); + expect(await readRun(runId)).toMatchObject({ status: "cancelled", nextEventSeq: 4, nativeSourceCursors: { "native-writer": 3 } }); + + const deletion = await holdRun(runId, async tx => { + await tx.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId)); + await tx.delete(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + }); + const late = appendHeartbeatRunEventBatch(db, [4, 5].map(sourceSeq => nativeEvent(runId, sourceSeq))); + const lateSingle = appendHeartbeatRunEvent(db, nativeEvent(runId, 4)); + const lateSettledWhileHeld = await settlesWhileHeld(late); + await deletion.release(); + expect(lateSettledWhileHeld).toBe(false); + await expect(late).rejects.toThrow("heartbeat_run_event_binding_mismatch"); + await expect(lateSingle).rejects.toThrow("heartbeat_run_event_binding_mismatch"); + expect(await readEvents(runId)).toEqual([]); + }); + + it("appends under the run lock when another writer commits events while the batch waits", async () => { + const runId = await makeRun(); + await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); + const older = await holdRun(runId, async tx => { + await tx.insert(heartbeatRunEvents).values([2, 3].map(sourceSeq => ({ + companyId, agentId, runId, seq: sourceSeq, eventType: "item.delta", stream: "system", level: "info", + payload: { prpEvent: { sourceSeq } }, sourceInstanceId: "native-writer", sourceEventId: `native-writer:${sourceSeq}`, + sourceSeq, sourcePayloadSha256: nativeSha256({ sourceSeq }), protocolSchemaVersion: 1, + }))); + await tx.update(heartbeatRuns).set({ nextEventSeq: 4 }).where(eq(heartbeatRuns.id, runId)); + }); + const batch = sentStatements(db.$client, () => appendHeartbeatRunEventBatch(db, [2, 3, 4].map(sourceSeq => nativeEvent(runId, sourceSeq)))); + const batchSettledWhileHeld = await settlesWhileHeld(batch); + await older.release(); + expect(batchSettledWhileHeld).toBe(false); + const { result, statements } = await batch; + expect(statements[0]).toMatch(/^with snapshot_run as /); + expect(statements[1]).toMatch(/^begin/i); + expect(result.map(receipt => [receipt.disposition, receipt.row.seq, receipt.highestContiguousSourceSeq])) + .toEqual([["duplicate", 2, 2], ["duplicate", 3, 3], ["committed", 4, 4]]); + expect((await readEvents(runId)).map(row => [row.seq, row.sourceSeq])).toEqual([[1, 1], [2, 2], [3, 3], [4, 4]]); + expect(await readRun(runId)).toMatchObject({ nextEventSeq: 5, nativeSourceCursors: { "native-writer": 4 } }); + }); + + it("keeps every source event once, in order, when two connections append batches of the same stream", async () => { + const runId = await makeRun(); + const stream = async (connection: typeof db) => { + const receipts = []; + for (let firstSeq = 1; firstSeq <= 200; firstSeq += 10) { + receipts.push(...await appendHeartbeatRunEventBatch(connection, + Array.from({ length: 10 }, (_, index) => nativeEvent(runId, firstSeq + index)))); + } + return receipts; + }; + const [ours, theirs] = await Promise.all([stream(db), stream(otherDb)]); + const rows = await readEvents(runId); + expect(rows.map(row => [row.seq, row.sourceSeq])).toEqual(Array.from({ length: 200 }, (_, index) => [index + 1, index + 1])); + expect(await readRun(runId)).toMatchObject({ nextEventSeq: 201, nativeSourceCursors: { "native-writer": 200 } }); + for (let index = 0; index < 200; index += 1) { + expect([ours[index]!.disposition, theirs[index]!.disposition].sort()).toEqual(["committed", "duplicate"]); + for (const receipt of [ours[index]!, theirs[index]!]) { + expect(receipt.row).toEqual(rows[index]); + expect(receipt.highestContiguousSourceSeq).toBeGreaterThanOrEqual(index + 1); + } + } + }); + + it("commits a batch larger than any bind parameter limit in one statement", async () => { + const runId = await makeRun(); + const batch = Array.from({ length: 7_000 }, (_, index) => nativeEvent(runId, index + 1)); + const appended = await sentStatements(db.$client, () => appendHeartbeatRunEventBatch(db, batch)); + expect(appended.statements).toHaveLength(1); + expect(appended.result.at(-1)).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 7_000, row: { seq: 7_000 } }); + const [stored] = await db.select({ count: sql`count(*)::int`, last: sql`max(seq)::int` }) + .from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId)); + expect(stored).toEqual({ count: 7_000, last: 7_000 }); + expect(await readRun(runId)).toMatchObject({ nextEventSeq: 7_001, nativeSourceCursors: { "native-writer": 7_000 } }); + }); + + it("starts a source without a cursor at its first event, and recounts a run that holds events from an older server", async () => { + const fresh = await makeRun(); + expect((await readRun(fresh)).nativeSourceCursors).toBeNull(); + const started = await sentStatements(db.$client, () => appendHeartbeatRunEventBatch(db, [1, 2].map(sourceSeq => nativeEvent(fresh, sourceSeq)))); + expect(started.statements).toHaveLength(1); + expect((await readRun(fresh)).nativeSourceCursors).toEqual({ "native-writer": 2 }); + + const older = await makeRun(); + await db.insert(heartbeatRunEvents).values([1, 2].map(sourceSeq => ({ + companyId, agentId, runId: older, seq: sourceSeq, eventType: "item.delta", stream: "system", level: "info", + payload: { prpEvent: { sourceSeq } }, sourceInstanceId: "native-writer", sourceEventId: `native-writer:${sourceSeq}`, + sourceSeq, sourcePayloadSha256: nativeSha256({ sourceSeq }), protocolSchemaVersion: 1, + }))); + await db.update(heartbeatRuns).set({ nextEventSeq: 3 }).where(eq(heartbeatRuns.id, older)); + const replayed = await appendHeartbeatRunEventBatch(db, [1, 2].map(sourceSeq => nativeEvent(older, sourceSeq))); + expect(replayed.map(receipt => [receipt.disposition, receipt.highestContiguousSourceSeq])).toEqual([["duplicate", 2], ["duplicate", 2]]); + const continued = await appendHeartbeatRunEventBatch(db, [3, 4].map(sourceSeq => nativeEvent(older, sourceSeq))); + expect(continued.map(receipt => [receipt.disposition, receipt.row.seq, receipt.highestContiguousSourceSeq])) + .toEqual([["committed", 3, 3], ["committed", 4, 4]]); + expect((await readRun(older)).nativeSourceCursors).toEqual({ "native-writer": 4 }); + }); + + it("produces the rows, receipts and cursor of the one-by-one append for random streams", async () => { + let state = 0x2f6b_1d3a; + const random = (limit: number) => { + state = (Math.imul(state ^ (state >>> 15), 0x2c1b3c6d) + 0x6d2b79f5) >>> 0; + return state % limit; + }; + const reference = (event: ReturnType) => ({ ...event, eventType: "turn.completed" }); + const appendBoth = async (fast: string, slow: string, sourceSeqs: number[], payloadOffset = 0) => { + const make = (runId: string, sourceSeq: number) => { + const event = nativeEvent(runId, sourceSeq); + return payloadOffset ? { ...event, nativeSource: { ...event.nativeSource, canonicalPayload: { sourceSeq: sourceSeq + payloadOffset } } } : event; + }; + const outcome = (receipts: Awaited>) => + receipts.map(receipt => [receipt.disposition, receipt.row.seq, receipt.row.sourceSeq, receipt.highestContiguousSourceSeq]); + const [ours, theirs] = await Promise.all([ + appendHeartbeatRunEventBatch(db, sourceSeqs.map(sourceSeq => make(fast, sourceSeq))).then(outcome, (error: Error) => error.message), + appendHeartbeatRunEventBatch(db, sourceSeqs.map(sourceSeq => reference(make(slow, sourceSeq)))).then(outcome, (error: Error) => error.message), + ]); + expect(ours).toEqual(theirs); + }; + let fastStatements = 0; + for (let stream = 0; stream < 12; stream += 1) { + const fast = await makeRun(), slow = await makeRun(); + let next = 1; + const sent: number[] = []; + for (let step = 0; step < 30; step += 1) { + const choice = random(10); + let sourceSeqs: number[]; + if (choice < 5) sourceSeqs = Array.from({ length: 1 + random(8) }, (_, index) => next + index); + else if (choice < 6) sourceSeqs = Array.from({ length: 1 + random(3) }, (_, index) => next + 1 + random(3) + index); + else if (choice < 8 && sent.length > 0) { + const start = sent[random(sent.length)]!; + sourceSeqs = Array.from({ length: 1 + random(5) }, (_, index) => start + index); + } else if (choice < 9) { + const missing = Array.from({ length: next - 1 }, (_, index) => index + 1).filter(sourceSeq => !sent.includes(sourceSeq)); + sourceSeqs = missing.length ? missing.slice(0, 1 + random(3)) : [next]; + } else { + for (const runId of [fast, slow]) await appendHeartbeatRunEvent(db, { companyId, agentId, runId, eventType: "stdout", message: `step-${step}` }); + if (sent.length > 0 && random(2) === 0) await appendBoth(fast, slow, [sent[random(sent.length)]!], 1_000); + continue; + } + const measured = await sentStatements(db.$client, () => appendBoth(fast, slow, sourceSeqs)); + fastStatements += measured.statements.filter(text => text.startsWith("with snapshot_run as")).length; + for (const sourceSeq of sourceSeqs) if (!sent.includes(sourceSeq)) sent.push(sourceSeq); + next = Math.max(next, ...sourceSeqs.map(sourceSeq => sourceSeq + 1)); + } + const project = (rows: Awaited>) => rows.map(row => + [row.seq, row.sourceInstanceId, row.sourceEventId, row.sourceSeq, row.sourcePayloadSha256, row.message]); + expect(project(await readEvents(fast))).toEqual(project(await readEvents(slow))); + const [fastRun, slowRun] = [await readRun(fast), await readRun(slow)]; + expect([fastRun.nextEventSeq, fastRun.nativeSourceCursors]).toEqual([slowRun.nextEventSeq, slowRun.nativeSourceCursors]); + } + expect(fastStatements).toBeGreaterThan(50); + }); + it("appends a batch one by one when its run has no cursor or the batch leaves a gap", async () => { const runId = await makeRun(); const fromStart = await appendHeartbeatRunEventBatch(db, [1, 2, 3].map(sourceSeq => nativeEvent(runId, sourceSeq))); @@ -336,10 +537,8 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { try { const before = await readRun(runId); await expect(appendHeartbeatRunEvent(db, failing)).rejects.toThrow(); - // A writer revoked while its append ran rolls the append back before the commit. - let checks = 0; await expect(appendHeartbeatRunEvent(db, { ...nativeEvent(runId, 3), - assertWritable: () => { if (++checks === 2) throw new Error("writer revoked"); } })).rejects.toThrow("writer revoked"); + assertWritable: () => { throw new Error("writer revoked"); } })).rejects.toThrow("writer revoked"); expect(await readRun(runId)).toEqual(before); expect((await readEvents(runId)).map(row => row.sourceSeq)).toEqual([1, 2]); // The stream continues past the failed event. Later events are stored diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index 7326396df3f..4aa41b431ff 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -6,10 +6,12 @@ import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "./na import { isSafeNativeChatProgressEventType } from "./safe-native-chat-progress.js"; import { notifyChatNativeProgress } from "./chat-native-progress-notifications.js"; import { boundRunIssueIdSql } from "./run-task-binding.js"; -import { and, asc, eq, getTableColumns, sql } from "drizzle-orm"; +import { and, asc, eq, getTableColumns, is, sql } from "drizzle-orm"; +import { PgTransaction } from "drizzle-orm/pg-core"; import type { Db } from "@paperclipai/db"; import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db"; import { nativeSha256 } from "./native-runtime/canonical.js"; +import { isUniqueViolation } from "../db-errors.js"; export const RETRY_EXHAUSTION_MESSAGE_PREFIX = "Bounded retry exhausted"; @@ -45,11 +47,12 @@ export interface AppendHeartbeatRunEventInput { */ runUpdate?: HeartbeatRunEventRunUpdate; /** - * Checks that the writer may still write to the run. It runs inside the - * append's transaction, before the first statement and again before the - * commit; when it throws, the event and its sequence roll back. A writer - * passes its check here instead of wrapping the append in a transaction of - * its own, which would add a SAVEPOINT round trip to every event. + * Checks that the writer may still write to the run. It runs before the + * first statement and again before the commit, or after an append that + * commits in its one statement. When it throws before the commit, the event + * and its sequence roll back; after it, the append stays and the caller gets + * the error instead of the receipt. A writer passes its check here instead of + * wrapping the append in a transaction of its own. */ assertWritable?: () => void; } @@ -139,6 +142,14 @@ export async function appendHeartbeatRunEvent( if (!appended) throw new Error("heartbeat_run_event_binding_mismatch"); return { row: decodeRunEventRow(appended), disposition: "committed", highestContiguousSourceSeq: 0 }; } + if (input.nativeSource && !input.runUpdate && !input.retryExhaustion && !Object.values(wakes).some(Boolean) && !is(db, PgTransaction)) { + input.assertWritable?.(); + const receipts = await appendNewNativeSourceEvents(db, [input as NativeSourceEventInput]); + if (receipts) { + input.assertWritable?.(); + return receipts[0]!; + } + } return db.transaction(async (tx) => { input.assertWritable?.(); const receipt = input.nativeSource @@ -150,13 +161,12 @@ export async function appendHeartbeatRunEvent( } /** - * Appends native source events of one run and one source in one transaction, - * in the given order, and returns one receipt per event. A stream that - * continues its source's stored cursor with new events takes four statements - * for the whole batch: the run lock, a replay probe, one insert of every row, - * and the commit. Any other batch (a replay, a gap, an event that registers a - * wake, or a run without a cursor) appends its events one by one in the same - * transaction, with the checks of a single append. + * Appends native source events of one run and one source atomically, in the + * given order, and returns one receipt per event. A stream that continues its + * source's stored cursor with new events takes one statement for the whole + * batch. Any other batch (a replay, a gap, an event that registers a wake, a + * run without a cursor, or a run another writer changed meanwhile) appends its + * events one by one in one transaction, with the checks of a single append. */ export async function appendHeartbeatRunEventBatch( db: Db, @@ -173,6 +183,11 @@ export async function appendHeartbeatRunEventBatch( } if (inputs.length === 1) return [await appendHeartbeatRunEvent(db, assertWritable ? { ...first, assertWritable } : first)]; const wakes = inputs.map(runEventWakes); + if (!wakes.some(wake => Object.values(wake).some(Boolean)) && !is(db, PgTransaction)) { + assertWritable?.(); + const receipts = await appendNewNativeSourceEvents(db, inputs as NativeSourceEventInput[]); + if (receipts) return finish(receipts); + } return db.transaction(async (transaction) => { const tx = transaction as unknown as Db; assertWritable?.(); @@ -245,6 +260,71 @@ export async function appendHeartbeatRunEventBatch( } } +const APPEND_NEW_NATIVE_SOURCE_EVENTS_SQL = ` + with snapshot_run as ( + select next_event_seq from heartbeat_runs where id = $1::uuid + ), allocated as ( + update heartbeat_runs + set next_event_seq = next_event_seq + $6::integer, + updated_at = $7::timestamptz, + native_source_cursors = jsonb_set(coalesce(native_source_cursors, '{}'::jsonb), + array[$4::text], to_jsonb($5::bigint + $6::integer - 1)) + where id = $1::uuid and company_id = $2::uuid and agent_id = $3::uuid + and coalesce((native_source_cursors ->> $4::text)::bigint, 0) = $5::bigint - 1 + and next_event_seq = (select next_event_seq from snapshot_run) + and not exists (select 1 from heartbeat_run_events where run_id = $1::uuid and ( + source_event_id = any(array(select jsonb_array_elements_text($8::jsonb))) + or (source_instance_id = $4::text and source_seq >= $5::bigint))) + returning next_event_seq - $6::integer as first_seq + ) + insert into heartbeat_run_events + (company_id, run_id, agent_id, seq, event_type, stream, level, color, message, payload, + source_instance_id, source_event_id, source_seq, source_payload_sha256, protocol_schema_version) + select $2::uuid, $1::uuid, $3::uuid, allocated.first_seq + event.ordinal - 1, + event.value ->> 'eventType', event.value ->> 'stream', event.value ->> 'level', event.value ->> 'color', + event.value ->> 'message', (event.value ->> 'payload')::jsonb, + $4::text, event.value ->> 'sourceEventId', $5::bigint - 1 + event.ordinal, + event.value ->> 'sourcePayloadSha256', (event.value ->> 'protocolSchemaVersion')::integer + from allocated cross join jsonb_array_elements($9::jsonb) with ordinality as event(value, ordinal) + order by event.ordinal + returning *`; + +async function appendNewNativeSourceEvents( + db: Db, + inputs: readonly NativeSourceEventInput[], +): Promise { + const first = inputs[0]!; + const source = first.nativeSource; + const firstSeq = source.sourceSeq; + if (typeof db.$client?.unsafe !== "function" || firstSeq < 1 + || inputs.some((input, index) => input.nativeSource.sourceSeq !== firstSeq + index)) return null; + const eventIds = JSON.stringify(inputs.map(input => input.nativeSource.sourceEventId)); + const rows = JSON.stringify(inputs.map(input => ({ + eventType: input.eventType, stream: input.stream ?? null, level: input.level ?? null, color: input.color ?? null, + message: input.message ?? null, + payload: input.payload ? heartbeatRunEvents.payload.mapToDriverValue(input.payload) : null, + sourceEventId: input.nativeSource.sourceEventId, + sourcePayloadSha256: nativeSha256(input.nativeSource.canonicalPayload), + protocolSchemaVersion: input.nativeSource.protocolSchemaVersion, + }))); + let inserted: Record[]; + try { + inserted = await db.$client.unsafe[]>(APPEND_NEW_NATIVE_SOURCE_EVENTS_SQL, [first.runId, first.companyId, + first.agentId, source.sourceInstanceId, String(firstSeq), String(inputs.length), new Date().toISOString(), eventIds, rows]); + } catch (error) { + if (isUniqueViolation(error)) return null; + throw error; + } + if (inserted.length === 0) return null; + if (inserted.length !== inputs.length) throw new Error("heartbeat_run_event_not_persisted"); + const bySourceSeq = new Map(inserted.map(raw => [Number(raw.source_seq), decodeRunEventRow(raw)])); + return inputs.map(input => ({ + row: bySourceSeq.get(input.nativeSource.sourceSeq)!, + disposition: "committed" as const, + highestContiguousSourceSeq: input.nativeSource.sourceSeq, + })); +} + const runEventColumns = Object.entries(getTableColumns(heartbeatRunEvents)); /** diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts index 385960d8cf0..e84571ffc71 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts @@ -920,7 +920,7 @@ describe("PaperclipControlPlanePort conformance", () => { expect(await readTaskPanelActivations(db, randomUUID(), identity.runId)).toEqual([]); }); - it("persists each streamed event in one transaction of two statements and rolls back a revoked writer", async () => { + it("persists each streamed event in one statement and rejects the receipt of a writer revoked while it commits", async () => { const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() }; const runnerId = randomUUID(); await db.insert(heartbeatRuns).values({ @@ -958,32 +958,33 @@ describe("PaperclipControlPlanePort conformance", () => { await expect(port.appendEvent(delta(sourceSeq))).resolves.toEqual({ cursor: sourceSeq, highestContiguousSourceSeq: sourceSeq, disposition: "committed", }); - // BEGIN, the run lock, the append, COMMIT: no SAVEPOINT and no separate - // replay, sequence, insert or cursor statement. - expect(statements).toHaveLength(4); - expect(statements.filter(text => /^(begin|commit)/i.test(text))).toHaveLength(2); + expect(statements).toHaveLength(1); + expect(statements[0]).toMatch(/^with snapshot_run as /); } statements.length = 0; await expect(port.appendEvent(delta(2))).resolves.toEqual({ cursor: 2, highestContiguousSourceSeq: 3, disposition: "duplicate" }); - expect(statements).toHaveLength(4); - // The controller loses the run while its append is in flight: the lock - // read is the first statement after the opening check. - options.debug = (_connection: number, text: string) => { if (/for no key update$/.test(text.trim())) active = false; }; + expect(statements).toHaveLength(5); + active = false; + statements.length = 0; + await expect(port.appendEvent(delta(4))).rejects.toThrow("controller_revoked"); + expect(statements).toEqual([]); + active = true; + options.debug = (_connection: number, text: string) => { if (/^with snapshot_run as /.test(text.trim())) active = false; }; await expect(port.appendEvent(delta(4))).rejects.toThrow("controller_revoked"); } finally { options.debug = undefined; } active = true; expect(committed).toEqual([1, 2, 3]); const stored = () => db.select({ seq: heartbeatRunEvents.seq, sourceSeq: heartbeatRunEvents.sourceSeq }) .from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, identity.runId)).orderBy(asc(heartbeatRunEvents.seq)); - expect(await stored()).toEqual([1, 2, 3].map(seq => ({ seq, sourceSeq: seq }))); - // The rolled-back event took no sequence: a replacement writes it next. + expect(await stored()).toEqual([1, 2, 3, 4].map(seq => ({ seq, sourceSeq: seq }))); const replacement = new PaperclipControlPlanePort(db, { ...identity, completionContractId: contractId, completionContractSha256: contractSha, sourceInstanceId: runnerId, controlPlaneSourceInstanceId: `stream-${identity.runId}`, }); await replacement.openRun({ identity, backendKind: "mock", sourceInstanceId: runnerId }); - await expect(replacement.appendEvent(delta(4))).resolves.toEqual({ cursor: 4, highestContiguousSourceSeq: 4, disposition: "committed" }); - expect(await stored()).toEqual([1, 2, 3, 4].map(seq => ({ seq, sourceSeq: seq }))); + await expect(replacement.appendEvent(delta(4))).resolves.toEqual({ cursor: 4, highestContiguousSourceSeq: 4, disposition: "duplicate" }); + await expect(replacement.appendEvent(delta(5))).resolves.toEqual({ cursor: 5, highestContiguousSourceSeq: 5, disposition: "committed" }); + expect(await stored()).toEqual([1, 2, 3, 4, 5].map(seq => ({ seq, sourceSeq: seq }))); }); const streamingFixture = async (options: ConstructorParameters[2] = {}) => { @@ -1041,9 +1042,10 @@ describe("PaperclipControlPlanePort conformance", () => { cursor: sourceSeq, highestContiguousSourceSeq: sourceSeq, disposition: "committed" }))); await expect(boundary).resolves.toMatchObject({ cursor: 7, highestContiguousSourceSeq: 7, disposition: "committed" }); expect(settled).toEqual([2, 3, 4, 5, 6]); - // The five deltas share one transaction and one insert; the boundary event has its own. - expect(statements.filter(text => text.includes("insert into heartbeat_run_events"))).toHaveLength(2); - expect(statements.filter(text => /^begin/i.test(text))).toHaveLength(2); + const inserts = statements.filter(text => text.includes("insert into heartbeat_run_events")); + expect(inserts).toHaveLength(2); + expect(statements[0]).toMatch(/^with snapshot_run as /); + expect(statements.indexOf(inserts[1]!)).toBeGreaterThan(0); } finally { options.debug = undefined; } expect(committed).toEqual([1, 2, 3, 4, 5, 6, 7]); expect(await f.stored()).toEqual([1, 2, 3, 4, 5, 6, 7].map(seq => ({ seq, sourceSeq: seq }))); diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.ts b/server/src/services/native-runtime/paperclip-control-plane-port.ts index c02b79b6b4b..bae8c88c4ad 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.ts @@ -463,8 +463,6 @@ export class PaperclipControlPlanePort implements ControlPlanePort { protocolSchemaVersion: event.schemaVersion, canonicalPayload: event as unknown as Record, }, - // The append has no publication side effects. Revocation rolls back - // its event and sequence allocation before the append commits. assertWritable: () => this.#assertActive(options), }; if (event.eventType === "item.delta" && !options?.deferPostCommitObservers) {