diff --git a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts index e13cb4fe461..e7da91ca4b7 100644 --- a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts +++ b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts @@ -130,26 +130,24 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { protocolSchemaVersion: 1, canonicalPayload: { sourceSeq } }, }); - it("appends a native source event with one lock read and one write statement", async () => { + it("appends an in-order native source event in one statement, and a replay under the run lock", async () => { const runId = await makeRun(); const first = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1))); - // BEGIN, the run lock, the append, COMMIT. The lock is a statement of its - // own so that the append reads a snapshot taken after the lock was granted. - expect(first.statements).toHaveLength(4); - expect(first.statements[0]).toMatch(/^begin/i); - expect(first.statements[1]).toMatch(/^select "company_id", "agent_id", \("native_source_cursors" ->> \$1\)::bigint as "source_cursor" from "heartbeat_runs" .* for no key update$/); - expect(first.statements[2]).toMatch(/^with existing as .* insert into heartbeat_run_events /); - expect(first.statements[3]).toMatch(/^commit/i); + expect(first.statements).toHaveLength(1); + expect(first.statements[0]).toMatch(/^with snapshot_run as .* update heartbeat_runs .* insert into heartbeat_run_events /); expect(first.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 1 }); expect(first.result.row.createdAt).toBeInstanceOf(Date); expect(await readEvents(runId)).toEqual([first.result.row]); const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1))); - expect(replay.statements).toHaveLength(4); + expect(replay.statements).toHaveLength(5); + expect(replay.statements[1]).toMatch(/^begin/i); + expect(replay.statements[2]).toMatch(/^select "company_id", "agent_id", \("native_source_cursors" ->> \$1\)::bigint as "source_cursor" from "heartbeat_runs" .* for no key update$/); + expect(replay.statements[3]).toMatch(/^with existing as .* insert into heartbeat_run_events /); + expect(replay.statements[4]).toMatch(/^commit/i); expect(replay.result).toEqual({ row: first.result.row, disposition: "duplicate", highestContiguousSourceSeq: 1 }); expect((await readRun(runId)).nextEventSeq).toBe(2); - // A caller's transaction adds only its SAVEPOINT; a writer check adds nothing. const nested = await sentStatements(db.$client, () => db.transaction(tx => appendHeartbeatRunEvent(tx as unknown as typeof db, nativeEvent(runId, 2)))); expect(nested.statements).toHaveLength(5); @@ -157,7 +155,7 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { const checked = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, { ...nativeEvent(runId, 3), assertWritable: () => { checks += 1; }, })); - expect(checked.statements).toHaveLength(4); + expect(checked.statements).toHaveLength(1); expect(checks).toBe(2); expect(checked.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 3 }); expect((await readEvents(runId)).map(row => [row.seq, row.sourceSeq])).toEqual([[1, 1], [2, 2], [3, 3]]); @@ -169,13 +167,13 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { for (let sourceSeq = 2; sourceSeq <= 6; sourceSeq += 1) { const appended = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, sourceSeq))); expect(appended.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: sourceSeq }); - expect(appended.statements).toHaveLength(4); + expect(appended.statements).toHaveLength(1); // The append statement reads no other event of the run. - expect(appended.statements[2]).not.toMatch(/count\(|max\(source_seq\)/); + expect(appended.statements[0]).not.toMatch(/count\(|max\(source_seq\)/); } const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 3))); expect(replay.result).toMatchObject({ disposition: "duplicate", highestContiguousSourceSeq: 6 }); - expect(replay.statements[2]).not.toMatch(/count\(/); + expect(replay.statements[3]).not.toMatch(/count\(/); expect((await readRun(runId)).nativeSourceCursors).toEqual({ "native-writer": 6 }); }); @@ -205,14 +203,13 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { expect((await readEvents(runId)).map(row => row.sourceSeq)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 10, 9]); }); - it("commits a batch of in-order deltas with one lock read, one replay probe and one insert", async () => { + it("commits a batch of in-order deltas in one statement", async () => { const runId = await makeRun(); await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); await appendHeartbeatRunEvent(db, { companyId, agentId, runId, eventType: "stdout", message: "ordinary" }); const batch = [2, 3, 4, 5, 6].map(sourceSeq => nativeEvent(runId, sourceSeq)); const appended = await sentStatements(db.$client, () => appendHeartbeatRunEventBatch(db, batch)); - // BEGIN, the run lock, the replay probe, one insert of every row, COMMIT. - expect(appended.statements).toHaveLength(5); + expect(appended.statements).toHaveLength(1); expect(appended.statements.filter(text => text.includes("insert into heartbeat_run_events"))).toHaveLength(1); expect(appended.result.map(receipt => [receipt.disposition, receipt.row.seq, receipt.row.sourceSeq, receipt.highestContiguousSourceSeq])) .toEqual([2, 3, 4, 5, 6].map((sourceSeq, index) => ["committed", index + 3, sourceSeq, sourceSeq])); @@ -254,10 +251,17 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); const before = await readRun(runId); const batch = [2, 3, 4].map(sourceSeq => nativeEvent(runId, sourceSeq)); - let checks = 0; - // The writer loses the run after the insert and before the commit. - await expect(appendHeartbeatRunEventBatch(db, batch, () => { if (++checks === 2) throw new Error("writer revoked"); })) - .rejects.toThrow("writer revoked"); + const revoked = await sentStatements(db.$client, () => appendHeartbeatRunEventBatch(db, batch, () => { + throw new Error("writer revoked"); + }).catch((error: Error) => error.message)); + expect(revoked).toEqual({ result: "writer revoked", statements: [] }); + await db.execute(sql`alter table heartbeat_run_events add constraint atomic_batch_failure + check (source_seq is distinct from 4) not valid`); + try { + await expect(appendHeartbeatRunEventBatch(db, batch)).rejects.toThrow(); + } finally { + await db.execute(sql`alter table heartbeat_run_events drop constraint atomic_batch_failure`); + } expect(await readRun(runId)).toEqual(before); expect((await readEvents(runId)).map(row => row.sourceSeq)).toEqual([1]); const resent = await appendHeartbeatRunEventBatch(db, batch); @@ -265,6 +269,203 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { .toEqual([["committed", 2, 2], ["committed", 3, 3], ["committed", 4, 4]]); }); + it("rejects the receipts of a writer revoked while its batch commits, and its resend is a replay", async () => { + const runId = await makeRun(); + await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); + const batch = [2, 3, 4].map(sourceSeq => nativeEvent(runId, sourceSeq)); + let checks = 0; + await expect(appendHeartbeatRunEventBatch(db, batch, () => { if (++checks === 2) throw new Error("writer revoked"); })) + .rejects.toThrow("writer revoked"); + expect((await readEvents(runId)).map(row => [row.seq, row.sourceSeq])).toEqual([[1, 1], [2, 2], [3, 3], [4, 4]]); + const resent = await appendHeartbeatRunEventBatch(db, batch); + expect(resent.map(receipt => [receipt.disposition, receipt.row.seq, receipt.highestContiguousSourceSeq])) + .toEqual([["duplicate", 2, 4], ["duplicate", 3, 4], ["duplicate", 4, 4]]); + expect(await readRun(runId)).toMatchObject({ nextEventSeq: 5, nativeSourceCursors: { "native-writer": 4 } }); + }); + + const holdRun = async (runId: string, work: (tx: typeof db) => Promise) => { + let release!: () => void, locked!: () => void; + const released = new Promise(resolve => { release = resolve; }); + const lockedPromise = new Promise(resolve => { locked = resolve; }); + const holder = otherDb.transaction(async tx => { + await tx.execute(sql`select id from heartbeat_runs where id = ${runId} for update`); + await work(tx as unknown as typeof db); + locked(); + await released; + }); + await lockedPromise; + return { release: () => { release(); return holder; } }; + }; + const settlesWhileHeld = async (pending: Promise) => { + let settled = false; + void pending.then(() => { settled = true; }, () => { settled = true; }); + await new Promise(resolve => setTimeout(resolve, 200)); + return settled; + }; + + it("waits for Stop on the run row and appends after it, and fails without rows after a run deletion", async () => { + const runId = await makeRun(); + await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); + const stop = await holdRun(runId, async tx => { + await tx.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, runId)); + }); + const batch = appendHeartbeatRunEventBatch(db, [2, 3].map(sourceSeq => nativeEvent(runId, sourceSeq))); + const batchSettledWhileHeld = await settlesWhileHeld(batch); + await stop.release(); + expect(batchSettledWhileHeld).toBe(false); + expect((await batch).map(receipt => [receipt.disposition, receipt.row.seq])).toEqual([["committed", 2], ["committed", 3]]); + expect(await readRun(runId)).toMatchObject({ status: "cancelled", nextEventSeq: 4, nativeSourceCursors: { "native-writer": 3 } }); + + const deletion = await holdRun(runId, async tx => { + await tx.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId)); + await tx.delete(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + }); + const late = appendHeartbeatRunEventBatch(db, [4, 5].map(sourceSeq => nativeEvent(runId, sourceSeq))); + const lateSingle = appendHeartbeatRunEvent(db, nativeEvent(runId, 4)); + const lateSettledWhileHeld = await settlesWhileHeld(late); + await deletion.release(); + expect(lateSettledWhileHeld).toBe(false); + await expect(late).rejects.toThrow("heartbeat_run_event_binding_mismatch"); + await expect(lateSingle).rejects.toThrow("heartbeat_run_event_binding_mismatch"); + expect(await readEvents(runId)).toEqual([]); + }); + + it("appends under the run lock when another writer commits events while the batch waits", async () => { + const runId = await makeRun(); + await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); + const older = await holdRun(runId, async tx => { + await tx.insert(heartbeatRunEvents).values([2, 3].map(sourceSeq => ({ + companyId, agentId, runId, seq: sourceSeq, eventType: "item.delta", stream: "system", level: "info", + payload: { prpEvent: { sourceSeq } }, sourceInstanceId: "native-writer", sourceEventId: `native-writer:${sourceSeq}`, + sourceSeq, sourcePayloadSha256: nativeSha256({ sourceSeq }), protocolSchemaVersion: 1, + }))); + await tx.update(heartbeatRuns).set({ nextEventSeq: 4 }).where(eq(heartbeatRuns.id, runId)); + }); + const batch = sentStatements(db.$client, () => appendHeartbeatRunEventBatch(db, [2, 3, 4].map(sourceSeq => nativeEvent(runId, sourceSeq)))); + const batchSettledWhileHeld = await settlesWhileHeld(batch); + await older.release(); + expect(batchSettledWhileHeld).toBe(false); + const { result, statements } = await batch; + expect(statements[0]).toMatch(/^with snapshot_run as /); + expect(statements[1]).toMatch(/^begin/i); + expect(result.map(receipt => [receipt.disposition, receipt.row.seq, receipt.highestContiguousSourceSeq])) + .toEqual([["duplicate", 2, 2], ["duplicate", 3, 3], ["committed", 4, 4]]); + expect((await readEvents(runId)).map(row => [row.seq, row.sourceSeq])).toEqual([[1, 1], [2, 2], [3, 3], [4, 4]]); + expect(await readRun(runId)).toMatchObject({ nextEventSeq: 5, nativeSourceCursors: { "native-writer": 4 } }); + }); + + it("keeps every source event once, in order, when two connections append batches of the same stream", async () => { + const runId = await makeRun(); + const stream = async (connection: typeof db) => { + const receipts = []; + for (let firstSeq = 1; firstSeq <= 200; firstSeq += 10) { + receipts.push(...await appendHeartbeatRunEventBatch(connection, + Array.from({ length: 10 }, (_, index) => nativeEvent(runId, firstSeq + index)))); + } + return receipts; + }; + const [ours, theirs] = await Promise.all([stream(db), stream(otherDb)]); + const rows = await readEvents(runId); + expect(rows.map(row => [row.seq, row.sourceSeq])).toEqual(Array.from({ length: 200 }, (_, index) => [index + 1, index + 1])); + expect(await readRun(runId)).toMatchObject({ nextEventSeq: 201, nativeSourceCursors: { "native-writer": 200 } }); + for (let index = 0; index < 200; index += 1) { + expect([ours[index]!.disposition, theirs[index]!.disposition].sort()).toEqual(["committed", "duplicate"]); + for (const receipt of [ours[index]!, theirs[index]!]) { + expect(receipt.row).toEqual(rows[index]); + expect(receipt.highestContiguousSourceSeq).toBeGreaterThanOrEqual(index + 1); + } + } + }); + + it("commits a batch larger than any bind parameter limit in one statement", async () => { + const runId = await makeRun(); + const batch = Array.from({ length: 7_000 }, (_, index) => nativeEvent(runId, index + 1)); + const appended = await sentStatements(db.$client, () => appendHeartbeatRunEventBatch(db, batch)); + expect(appended.statements).toHaveLength(1); + expect(appended.result.at(-1)).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 7_000, row: { seq: 7_000 } }); + const [stored] = await db.select({ count: sql`count(*)::int`, last: sql`max(seq)::int` }) + .from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, runId)); + expect(stored).toEqual({ count: 7_000, last: 7_000 }); + expect(await readRun(runId)).toMatchObject({ nextEventSeq: 7_001, nativeSourceCursors: { "native-writer": 7_000 } }); + }); + + it("starts a source without a cursor at its first event, and recounts a run that holds events from an older server", async () => { + const fresh = await makeRun(); + expect((await readRun(fresh)).nativeSourceCursors).toBeNull(); + const started = await sentStatements(db.$client, () => appendHeartbeatRunEventBatch(db, [1, 2].map(sourceSeq => nativeEvent(fresh, sourceSeq)))); + expect(started.statements).toHaveLength(1); + expect((await readRun(fresh)).nativeSourceCursors).toEqual({ "native-writer": 2 }); + + const older = await makeRun(); + await db.insert(heartbeatRunEvents).values([1, 2].map(sourceSeq => ({ + companyId, agentId, runId: older, seq: sourceSeq, eventType: "item.delta", stream: "system", level: "info", + payload: { prpEvent: { sourceSeq } }, sourceInstanceId: "native-writer", sourceEventId: `native-writer:${sourceSeq}`, + sourceSeq, sourcePayloadSha256: nativeSha256({ sourceSeq }), protocolSchemaVersion: 1, + }))); + await db.update(heartbeatRuns).set({ nextEventSeq: 3 }).where(eq(heartbeatRuns.id, older)); + const replayed = await appendHeartbeatRunEventBatch(db, [1, 2].map(sourceSeq => nativeEvent(older, sourceSeq))); + expect(replayed.map(receipt => [receipt.disposition, receipt.highestContiguousSourceSeq])).toEqual([["duplicate", 2], ["duplicate", 2]]); + const continued = await appendHeartbeatRunEventBatch(db, [3, 4].map(sourceSeq => nativeEvent(older, sourceSeq))); + expect(continued.map(receipt => [receipt.disposition, receipt.row.seq, receipt.highestContiguousSourceSeq])) + .toEqual([["committed", 3, 3], ["committed", 4, 4]]); + expect((await readRun(older)).nativeSourceCursors).toEqual({ "native-writer": 4 }); + }); + + it("produces the rows, receipts and cursor of the one-by-one append for random streams", async () => { + let state = 0x2f6b_1d3a; + const random = (limit: number) => { + state = (Math.imul(state ^ (state >>> 15), 0x2c1b3c6d) + 0x6d2b79f5) >>> 0; + return state % limit; + }; + const reference = (event: ReturnType) => ({ ...event, eventType: "turn.completed" }); + const appendBoth = async (fast: string, slow: string, sourceSeqs: number[], payloadOffset = 0) => { + const make = (runId: string, sourceSeq: number) => { + const event = nativeEvent(runId, sourceSeq); + return payloadOffset ? { ...event, nativeSource: { ...event.nativeSource, canonicalPayload: { sourceSeq: sourceSeq + payloadOffset } } } : event; + }; + const outcome = (receipts: Awaited>) => + receipts.map(receipt => [receipt.disposition, receipt.row.seq, receipt.row.sourceSeq, receipt.highestContiguousSourceSeq]); + const [ours, theirs] = await Promise.all([ + appendHeartbeatRunEventBatch(db, sourceSeqs.map(sourceSeq => make(fast, sourceSeq))).then(outcome, (error: Error) => error.message), + appendHeartbeatRunEventBatch(db, sourceSeqs.map(sourceSeq => reference(make(slow, sourceSeq)))).then(outcome, (error: Error) => error.message), + ]); + expect(ours).toEqual(theirs); + }; + let fastStatements = 0; + for (let stream = 0; stream < 12; stream += 1) { + const fast = await makeRun(), slow = await makeRun(); + let next = 1; + const sent: number[] = []; + for (let step = 0; step < 30; step += 1) { + const choice = random(10); + let sourceSeqs: number[]; + if (choice < 5) sourceSeqs = Array.from({ length: 1 + random(8) }, (_, index) => next + index); + else if (choice < 6) sourceSeqs = Array.from({ length: 1 + random(3) }, (_, index) => next + 1 + random(3) + index); + else if (choice < 8 && sent.length > 0) { + const start = sent[random(sent.length)]!; + sourceSeqs = Array.from({ length: 1 + random(5) }, (_, index) => start + index); + } else if (choice < 9) { + const missing = Array.from({ length: next - 1 }, (_, index) => index + 1).filter(sourceSeq => !sent.includes(sourceSeq)); + sourceSeqs = missing.length ? missing.slice(0, 1 + random(3)) : [next]; + } else { + for (const runId of [fast, slow]) await appendHeartbeatRunEvent(db, { companyId, agentId, runId, eventType: "stdout", message: `step-${step}` }); + if (sent.length > 0 && random(2) === 0) await appendBoth(fast, slow, [sent[random(sent.length)]!], 1_000); + continue; + } + const measured = await sentStatements(db.$client, () => appendBoth(fast, slow, sourceSeqs)); + fastStatements += measured.statements.filter(text => text.startsWith("with snapshot_run as")).length; + for (const sourceSeq of sourceSeqs) if (!sent.includes(sourceSeq)) sent.push(sourceSeq); + next = Math.max(next, ...sourceSeqs.map(sourceSeq => sourceSeq + 1)); + } + const project = (rows: Awaited>) => rows.map(row => + [row.seq, row.sourceInstanceId, row.sourceEventId, row.sourceSeq, row.sourcePayloadSha256, row.message]); + expect(project(await readEvents(fast))).toEqual(project(await readEvents(slow))); + const [fastRun, slowRun] = [await readRun(fast), await readRun(slow)]; + expect([fastRun.nextEventSeq, fastRun.nativeSourceCursors]).toEqual([slowRun.nextEventSeq, slowRun.nativeSourceCursors]); + } + expect(fastStatements).toBeGreaterThan(50); + }); + it("appends a batch one by one when its run has no cursor or the batch leaves a gap", async () => { const runId = await makeRun(); const fromStart = await appendHeartbeatRunEventBatch(db, [1, 2, 3].map(sourceSeq => nativeEvent(runId, sourceSeq))); @@ -336,10 +537,8 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { try { const before = await readRun(runId); await expect(appendHeartbeatRunEvent(db, failing)).rejects.toThrow(); - // A writer revoked while its append ran rolls the append back before the commit. - let checks = 0; await expect(appendHeartbeatRunEvent(db, { ...nativeEvent(runId, 3), - assertWritable: () => { if (++checks === 2) throw new Error("writer revoked"); } })).rejects.toThrow("writer revoked"); + assertWritable: () => { throw new Error("writer revoked"); } })).rejects.toThrow("writer revoked"); expect(await readRun(runId)).toEqual(before); expect((await readEvents(runId)).map(row => row.sourceSeq)).toEqual([1, 2]); // The stream continues past the failed event. Later events are stored diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index 7326396df3f..4aa41b431ff 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -6,10 +6,12 @@ import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "./na import { isSafeNativeChatProgressEventType } from "./safe-native-chat-progress.js"; import { notifyChatNativeProgress } from "./chat-native-progress-notifications.js"; import { boundRunIssueIdSql } from "./run-task-binding.js"; -import { and, asc, eq, getTableColumns, sql } from "drizzle-orm"; +import { and, asc, eq, getTableColumns, is, sql } from "drizzle-orm"; +import { PgTransaction } from "drizzle-orm/pg-core"; import type { Db } from "@paperclipai/db"; import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db"; import { nativeSha256 } from "./native-runtime/canonical.js"; +import { isUniqueViolation } from "../db-errors.js"; export const RETRY_EXHAUSTION_MESSAGE_PREFIX = "Bounded retry exhausted"; @@ -45,11 +47,12 @@ export interface AppendHeartbeatRunEventInput { */ runUpdate?: HeartbeatRunEventRunUpdate; /** - * Checks that the writer may still write to the run. It runs inside the - * append's transaction, before the first statement and again before the - * commit; when it throws, the event and its sequence roll back. A writer - * passes its check here instead of wrapping the append in a transaction of - * its own, which would add a SAVEPOINT round trip to every event. + * Checks that the writer may still write to the run. It runs before the + * first statement and again before the commit, or after an append that + * commits in its one statement. When it throws before the commit, the event + * and its sequence roll back; after it, the append stays and the caller gets + * the error instead of the receipt. A writer passes its check here instead of + * wrapping the append in a transaction of its own. */ assertWritable?: () => void; } @@ -139,6 +142,14 @@ export async function appendHeartbeatRunEvent( if (!appended) throw new Error("heartbeat_run_event_binding_mismatch"); return { row: decodeRunEventRow(appended), disposition: "committed", highestContiguousSourceSeq: 0 }; } + if (input.nativeSource && !input.runUpdate && !input.retryExhaustion && !Object.values(wakes).some(Boolean) && !is(db, PgTransaction)) { + input.assertWritable?.(); + const receipts = await appendNewNativeSourceEvents(db, [input as NativeSourceEventInput]); + if (receipts) { + input.assertWritable?.(); + return receipts[0]!; + } + } return db.transaction(async (tx) => { input.assertWritable?.(); const receipt = input.nativeSource @@ -150,13 +161,12 @@ export async function appendHeartbeatRunEvent( } /** - * Appends native source events of one run and one source in one transaction, - * in the given order, and returns one receipt per event. A stream that - * continues its source's stored cursor with new events takes four statements - * for the whole batch: the run lock, a replay probe, one insert of every row, - * and the commit. Any other batch (a replay, a gap, an event that registers a - * wake, or a run without a cursor) appends its events one by one in the same - * transaction, with the checks of a single append. + * Appends native source events of one run and one source atomically, in the + * given order, and returns one receipt per event. A stream that continues its + * source's stored cursor with new events takes one statement for the whole + * batch. Any other batch (a replay, a gap, an event that registers a wake, a + * run without a cursor, or a run another writer changed meanwhile) appends its + * events one by one in one transaction, with the checks of a single append. */ export async function appendHeartbeatRunEventBatch( db: Db, @@ -173,6 +183,11 @@ export async function appendHeartbeatRunEventBatch( } if (inputs.length === 1) return [await appendHeartbeatRunEvent(db, assertWritable ? { ...first, assertWritable } : first)]; const wakes = inputs.map(runEventWakes); + if (!wakes.some(wake => Object.values(wake).some(Boolean)) && !is(db, PgTransaction)) { + assertWritable?.(); + const receipts = await appendNewNativeSourceEvents(db, inputs as NativeSourceEventInput[]); + if (receipts) return finish(receipts); + } return db.transaction(async (transaction) => { const tx = transaction as unknown as Db; assertWritable?.(); @@ -245,6 +260,71 @@ export async function appendHeartbeatRunEventBatch( } } +const APPEND_NEW_NATIVE_SOURCE_EVENTS_SQL = ` + with snapshot_run as ( + select next_event_seq from heartbeat_runs where id = $1::uuid + ), allocated as ( + update heartbeat_runs + set next_event_seq = next_event_seq + $6::integer, + updated_at = $7::timestamptz, + native_source_cursors = jsonb_set(coalesce(native_source_cursors, '{}'::jsonb), + array[$4::text], to_jsonb($5::bigint + $6::integer - 1)) + where id = $1::uuid and company_id = $2::uuid and agent_id = $3::uuid + and coalesce((native_source_cursors ->> $4::text)::bigint, 0) = $5::bigint - 1 + and next_event_seq = (select next_event_seq from snapshot_run) + and not exists (select 1 from heartbeat_run_events where run_id = $1::uuid and ( + source_event_id = any(array(select jsonb_array_elements_text($8::jsonb))) + or (source_instance_id = $4::text and source_seq >= $5::bigint))) + returning next_event_seq - $6::integer as first_seq + ) + insert into heartbeat_run_events + (company_id, run_id, agent_id, seq, event_type, stream, level, color, message, payload, + source_instance_id, source_event_id, source_seq, source_payload_sha256, protocol_schema_version) + select $2::uuid, $1::uuid, $3::uuid, allocated.first_seq + event.ordinal - 1, + event.value ->> 'eventType', event.value ->> 'stream', event.value ->> 'level', event.value ->> 'color', + event.value ->> 'message', (event.value ->> 'payload')::jsonb, + $4::text, event.value ->> 'sourceEventId', $5::bigint - 1 + event.ordinal, + event.value ->> 'sourcePayloadSha256', (event.value ->> 'protocolSchemaVersion')::integer + from allocated cross join jsonb_array_elements($9::jsonb) with ordinality as event(value, ordinal) + order by event.ordinal + returning *`; + +async function appendNewNativeSourceEvents( + db: Db, + inputs: readonly NativeSourceEventInput[], +): Promise { + const first = inputs[0]!; + const source = first.nativeSource; + const firstSeq = source.sourceSeq; + if (typeof db.$client?.unsafe !== "function" || firstSeq < 1 + || inputs.some((input, index) => input.nativeSource.sourceSeq !== firstSeq + index)) return null; + const eventIds = JSON.stringify(inputs.map(input => input.nativeSource.sourceEventId)); + const rows = JSON.stringify(inputs.map(input => ({ + eventType: input.eventType, stream: input.stream ?? null, level: input.level ?? null, color: input.color ?? null, + message: input.message ?? null, + payload: input.payload ? heartbeatRunEvents.payload.mapToDriverValue(input.payload) : null, + sourceEventId: input.nativeSource.sourceEventId, + sourcePayloadSha256: nativeSha256(input.nativeSource.canonicalPayload), + protocolSchemaVersion: input.nativeSource.protocolSchemaVersion, + }))); + let inserted: Record[]; + try { + inserted = await db.$client.unsafe[]>(APPEND_NEW_NATIVE_SOURCE_EVENTS_SQL, [first.runId, first.companyId, + first.agentId, source.sourceInstanceId, String(firstSeq), String(inputs.length), new Date().toISOString(), eventIds, rows]); + } catch (error) { + if (isUniqueViolation(error)) return null; + throw error; + } + if (inserted.length === 0) return null; + if (inserted.length !== inputs.length) throw new Error("heartbeat_run_event_not_persisted"); + const bySourceSeq = new Map(inserted.map(raw => [Number(raw.source_seq), decodeRunEventRow(raw)])); + return inputs.map(input => ({ + row: bySourceSeq.get(input.nativeSource.sourceSeq)!, + disposition: "committed" as const, + highestContiguousSourceSeq: input.nativeSource.sourceSeq, + })); +} + const runEventColumns = Object.entries(getTableColumns(heartbeatRunEvents)); /** diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts index 385960d8cf0..e84571ffc71 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts @@ -920,7 +920,7 @@ describe("PaperclipControlPlanePort conformance", () => { expect(await readTaskPanelActivations(db, randomUUID(), identity.runId)).toEqual([]); }); - it("persists each streamed event in one transaction of two statements and rolls back a revoked writer", async () => { + it("persists each streamed event in one statement and rejects the receipt of a writer revoked while it commits", async () => { const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() }; const runnerId = randomUUID(); await db.insert(heartbeatRuns).values({ @@ -958,32 +958,33 @@ describe("PaperclipControlPlanePort conformance", () => { await expect(port.appendEvent(delta(sourceSeq))).resolves.toEqual({ cursor: sourceSeq, highestContiguousSourceSeq: sourceSeq, disposition: "committed", }); - // BEGIN, the run lock, the append, COMMIT: no SAVEPOINT and no separate - // replay, sequence, insert or cursor statement. - expect(statements).toHaveLength(4); - expect(statements.filter(text => /^(begin|commit)/i.test(text))).toHaveLength(2); + expect(statements).toHaveLength(1); + expect(statements[0]).toMatch(/^with snapshot_run as /); } statements.length = 0; await expect(port.appendEvent(delta(2))).resolves.toEqual({ cursor: 2, highestContiguousSourceSeq: 3, disposition: "duplicate" }); - expect(statements).toHaveLength(4); - // The controller loses the run while its append is in flight: the lock - // read is the first statement after the opening check. - options.debug = (_connection: number, text: string) => { if (/for no key update$/.test(text.trim())) active = false; }; + expect(statements).toHaveLength(5); + active = false; + statements.length = 0; + await expect(port.appendEvent(delta(4))).rejects.toThrow("controller_revoked"); + expect(statements).toEqual([]); + active = true; + options.debug = (_connection: number, text: string) => { if (/^with snapshot_run as /.test(text.trim())) active = false; }; await expect(port.appendEvent(delta(4))).rejects.toThrow("controller_revoked"); } finally { options.debug = undefined; } active = true; expect(committed).toEqual([1, 2, 3]); const stored = () => db.select({ seq: heartbeatRunEvents.seq, sourceSeq: heartbeatRunEvents.sourceSeq }) .from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, identity.runId)).orderBy(asc(heartbeatRunEvents.seq)); - expect(await stored()).toEqual([1, 2, 3].map(seq => ({ seq, sourceSeq: seq }))); - // The rolled-back event took no sequence: a replacement writes it next. + expect(await stored()).toEqual([1, 2, 3, 4].map(seq => ({ seq, sourceSeq: seq }))); const replacement = new PaperclipControlPlanePort(db, { ...identity, completionContractId: contractId, completionContractSha256: contractSha, sourceInstanceId: runnerId, controlPlaneSourceInstanceId: `stream-${identity.runId}`, }); await replacement.openRun({ identity, backendKind: "mock", sourceInstanceId: runnerId }); - await expect(replacement.appendEvent(delta(4))).resolves.toEqual({ cursor: 4, highestContiguousSourceSeq: 4, disposition: "committed" }); - expect(await stored()).toEqual([1, 2, 3, 4].map(seq => ({ seq, sourceSeq: seq }))); + await expect(replacement.appendEvent(delta(4))).resolves.toEqual({ cursor: 4, highestContiguousSourceSeq: 4, disposition: "duplicate" }); + await expect(replacement.appendEvent(delta(5))).resolves.toEqual({ cursor: 5, highestContiguousSourceSeq: 5, disposition: "committed" }); + expect(await stored()).toEqual([1, 2, 3, 4, 5].map(seq => ({ seq, sourceSeq: seq }))); }); const streamingFixture = async (options: ConstructorParameters[2] = {}) => { @@ -1041,9 +1042,10 @@ describe("PaperclipControlPlanePort conformance", () => { cursor: sourceSeq, highestContiguousSourceSeq: sourceSeq, disposition: "committed" }))); await expect(boundary).resolves.toMatchObject({ cursor: 7, highestContiguousSourceSeq: 7, disposition: "committed" }); expect(settled).toEqual([2, 3, 4, 5, 6]); - // The five deltas share one transaction and one insert; the boundary event has its own. - expect(statements.filter(text => text.includes("insert into heartbeat_run_events"))).toHaveLength(2); - expect(statements.filter(text => /^begin/i.test(text))).toHaveLength(2); + const inserts = statements.filter(text => text.includes("insert into heartbeat_run_events")); + expect(inserts).toHaveLength(2); + expect(statements[0]).toMatch(/^with snapshot_run as /); + expect(statements.indexOf(inserts[1]!)).toBeGreaterThan(0); } finally { options.debug = undefined; } expect(committed).toEqual([1, 2, 3, 4, 5, 6, 7]); expect(await f.stored()).toEqual([1, 2, 3, 4, 5, 6, 7].map(seq => ({ seq, sourceSeq: seq }))); diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.ts b/server/src/services/native-runtime/paperclip-control-plane-port.ts index c02b79b6b4b..bae8c88c4ad 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.ts @@ -463,8 +463,6 @@ export class PaperclipControlPlanePort implements ControlPlanePort { protocolSchemaVersion: event.schemaVersion, canonicalPayload: event as unknown as Record, }, - // The append has no publication side effects. Revocation rolls back - // its event and sequence allocation before the append commits. assertWritable: () => this.#assertActive(options), }; if (event.eventType === "item.delta" && !options?.deferPostCommitObservers) {