diff --git a/packages/paperclip-runner/src/contracts/control-plane-port.ts b/packages/paperclip-runner/src/contracts/control-plane-port.ts index 6b91bfd4492..1e7b5664a7f 100644 --- a/packages/paperclip-runner/src/contracts/control-plane-port.ts +++ b/packages/paperclip-runner/src/contracts/control-plane-port.ts @@ -35,6 +35,12 @@ export interface AppendControlPlaneEventOptions { * acceptance. Ports that do not support deferral keep their usual behavior. */ deferPostCommitObservers?: true; + /** + * For an `item.delta` only, on a port with `pipelinesStreamingEvents`: the + * caller does not wait for this receipt before its next append, so the port + * may hold the delta briefly to commit it with later ones. + */ + streaming?: true; } export interface CheckpointControlPlaneSessionOptions { @@ -80,6 +86,17 @@ export interface CompleteControlPlaneRunInput { * runner package never imports the server, UI, or database that sits behind it. */ export interface ControlPlanePort { + /** + * True when the port commits `item.delta` events in batches. A caller may + * then append a delta with `streaming` and start the next append before its + * receipt settles. The port + * commits every other event, checkpoint and result only after every delta + * appended before it, and a delta's receipt settles only after its batch + * commits. + */ + readonly pipelinesStreamingEvents?: boolean; + /** Commits the queued streaming deltas now, without waiting for the batch timer. */ + flushStreamingEvents?(): void; normalizeReplayPayload?(payload: PrpEvent["payload"]): PrpEvent["payload"]; openRun(input: OpenControlPlaneRunInput): Promise; loadSessionCheckpoint?(): Promise; diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index ec48e18204a..93813aab14b 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -311,6 +311,100 @@ describe("executeNativeSession recovery", () => { } finally { abort.abort(); await completion.catch(() => undefined); } }); + /** A batching port whose queued deltas commit only when flushed: it has no timer. */ + const batchingDeltaFixture = (events: () => AsyncGenerator, commitDelta: (sourceSeq: number) => Promise = async () => {}) => { + const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + const session: NativeSession = { + identity: () => identity, + async capabilities() { return capabilities; }, + events, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot() { + return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", + cursor: null, activeTurnId: null, semanticResult: null, terminal: null, terminalTurns: [], pendingRuntimeRequests: [], lineage: [] }; + }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { return { kind: "mock", name: "delta-fixture", version: "1", capabilities, + runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" } }; }, + async openSession() { return session; }, + }; + const log: string[] = []; + let queued: Array<() => void> = []; + const port: ControlPlanePort = { + pipelinesStreamingEvents: true, + flushStreamingEvents() { + log.push("flush"); + const flushed = queued; + queued = []; + for (const commit of flushed) commit(); + }, + async openRun() {}, async checkpointSession() {}, + async appendEvent(event: PrpEvent, options) { + if (event.sourceKind !== "runner") return { cursor: 0, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; + log.push(`append:${event.sourceSeq}`); + if (event.eventType === "item.delta") { + expect(options?.streaming).toBe(true); + await new Promise(resolve => queued.push(resolve)); + await commitDelta(event.sourceSeq); + log.push(`committed:${event.sourceSeq}`); + } + return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + return { backend, port, log }; + }; + const streamedDelta = (sourceSeq: number) => ({ ...runnerEvent(sourceSeq, "item.delta", { kind: "agentMessage", text: `chunk ${sourceSeq}` }), + itemId: "message-1", priority: 1 as const }); + + it("starts text delta appends without waiting and flushes them before any other event", async () => { + const f = batchingDeltaFixture(async function* () { + for (const sourceSeq of [1, 2, 3]) yield streamedDelta(sourceSeq); + yield runnerEvent(4, "run.result.proposed", result); + yield runnerEvent(5, "turn.completed"); + }); + // The port has no timer: the turn finishes only if the next event flushes the queued deltas. + const execution = await executeNativeSession({ input: preparedInput(), backend: f.backend, controlPlane: f.port, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }); + expect(f.log.slice(0, 5)).toEqual(["append:1", "append:2", "append:3", "flush", "committed:1"]); + expect(f.log.indexOf("append:4")).toBeGreaterThan(Math.max(...[1, 2, 3].map(seq => f.log.indexOf(`committed:${seq}`)))); + expect(execution).toMatchObject({ nativeEventCount: 7, highestContiguousSourceSeq: 5 }); + }); + + it("joins pending delta appends before a failed turn settles", async () => { + let releaseCommit = () => {}; + const commitReleased = new Promise(resolve => { releaseCommit = resolve; }); + const f = batchingDeltaFixture(async function* () { + yield streamedDelta(1); + throw new Error("provider stream broke"); + }, () => commitReleased); + let settled = false; + const execution = executeNativeSession({ input: preparedInput(), backend: f.backend, controlPlane: f.port, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }).finally(() => { settled = true; f.log.push("settled"); }); + void execution.catch(() => undefined); + await vi.waitFor(() => expect(f.log).toContain("flush")); + // The delta's commit still runs, so the turn has not settled. + expect(settled).toBe(false); + releaseCommit(); + await expect(execution).rejects.toThrow("provider stream broke"); + expect(f.log.indexOf("committed:1")).toBeLessThan(f.log.indexOf("settled")); + }); + + it("fails the turn within the cleanup bound when a pending delta append never settles", async () => { + const f = batchingDeltaFixture(async function* () { + yield streamedDelta(1); + throw new Error("provider stream broke"); + }, () => new Promise(() => {})); + // No turn timeout: only the bounded cleanup can end this turn. + await expect(executeNativeSession({ input: preparedInput(), backend: f.backend, controlPlane: f.port, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" })).rejects.toThrow("provider stream broke"); + expect(f.log).not.toContain("committed:1"); + }); + it.each([false, true].flatMap(recovered => [ { recovered, cwd: "/home/agent" }, { recovered, cwd: "/projects/repository/worktrees/task" }, diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 4bbd09856cb..bd54e2f0d7d 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { nativeSystemInstructions } from "./backends/runtime-context.js"; import type { + AppendedEventReceipt, CheckpointControlPlaneSessionOptions, ControlPlanePort, } from "./contracts/control-plane-port.js"; @@ -65,6 +66,8 @@ const REUSABLE_SESSION_CANCELLATION_SETTLEMENT_GRACE_MS = 10_000; // elapsed time or a cancellation acknowledgement as usage/terminal evidence. const GOVERNED_ACCOUNTING_DRAIN_MS = 15_000; const MAX_GOVERNED_ACCOUNTING_DRAIN_MS = 30_000; +// Text delta receipts a turn may leave unsettled before it waits for them. +const MAX_PENDING_DELTA_RECEIPTS = 256; const DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS = 30_000; /** * A retained cleanup owner can only still hold resources it was bound to. @@ -1228,7 +1231,16 @@ async function consumeTurn( if (inputTimer !== undefined) clearTimeout(inputTimer); inputTimers.delete(requestId); }; - const consumer = (async () => { + // Delta appends the consumer started and that have not settled. The consumer + // joins them before it settles, whatever way it ends, so their observers + // never outlive the turn's cleanup. + const outstandingDeltas = new Set>(); + const joinOutstandingDeltas = async () => { + if (outstandingDeltas.size === 0) return; + controlPlane.flushStreamingEvents?.(); + await Promise.allSettled([...outstandingDeltas]); + }; + const consumerBody = (async () => { let eventCount = 0; let highestContiguousSourceSeq = 0; let governedResult: PrpStructuredRunResult | null = null; @@ -1239,11 +1251,30 @@ async function consumeTurn( let latestSessionGoal: HarnessThreadGoal | null = null; let resultSource: "semantic_result" | "governed_wait" | null = null; let providerFailure: NativeProviderTerminalFailure | null = null; + // A port that batches text deltas lets the next event start before a + // delta's receipt settles. Every other event, and every result, waits for + // the receipts of the deltas before it. + const pipelinesDeltas = controlPlane.pipelinesStreamingEvents === true; + let pendingDeltas: Promise[] = []; + let deltaFailure = null as { error: unknown } | null; + const settlePendingDeltas = async () => { + // Queued deltas commit now: the next event must not wait for the batch timer. + if (pendingDeltas.length > 0) controlPlane.flushStreamingEvents?.(); + while (pendingDeltas.length > 0) { + const pending = pendingDeltas; + pendingDeltas = []; + for (const receipt of await Promise.all(pending)) { + eventCount += receipt.disposition === "committed" ? 1 : 0; + highestContiguousSourceSeq = Math.max(highestContiguousSourceSeq, receipt.highestContiguousSourceSeq); + } + } + }; const settleDurableResult = async ( event: PrpEvent, result: PrpStructuredRunResult, reason: string, ) => { + await settlePendingDeltas(); if (session.cancel === undefined) { throw new Error("native_governed_wait_cancellation_unavailable"); } @@ -1384,15 +1415,30 @@ async function consumeTurn( // the timer stayed live until the receipt returned, both settlement // and a durable handoff could commit for the same request. if (settlingRequestId !== null) clearInputTimer(settlingRequestId); - const receipt = await controlPlane.appendEvent(event, { - signal: appendAbort.signal, - }); + // A failed delta stops the turn before any later event is appended. + if (deltaFailure) throw deltaFailure.error; + let receipt: AppendedEventReceipt | null = null; + if (pipelinesDeltas && event.eventType === "item.delta") { + const pending = controlPlane.appendEvent(event, { signal: appendAbort.signal, streaming: true }); + const settled = pending.then(() => undefined, (error: unknown) => { deltaFailure ??= { error }; }); + outstandingDeltas.add(settled); + void settled.finally(() => outstandingDeltas.delete(settled)); + pendingDeltas.push(pending); + if (pendingDeltas.length >= MAX_PENDING_DELTA_RECEIPTS) await settlePendingDeltas(); + } else { + await settlePendingDeltas(); + receipt = await controlPlane.appendEvent(event, { + signal: appendAbort.signal, + }); + } if (stopConsumer) throw new Error("native event consumer stopped"); - eventCount += receipt.disposition === "committed" ? 1 : 0; - highestContiguousSourceSeq = Math.max( - highestContiguousSourceSeq, - receipt.highestContiguousSourceSeq, - ); + if (receipt !== null) { + eventCount += receipt.disposition === "committed" ? 1 : 0; + highestContiguousSourceSeq = Math.max( + highestContiguousSourceSeq, + receipt.highestContiguousSourceSeq, + ); + } const eventGoal = goalFromEvent(event); const goalChanged = eventGoal !== undefined && goalLifecycleFingerprint(eventGoal) !== goalLifecycleFingerprint(previousGoal); @@ -1430,7 +1476,7 @@ async function consumeTurn( typeof failure.message === "string" ? failure.message : undefined, readProviderFailure(failure.providerFailure), ); - if ((receipt.disposition === "committed" || receipt.disposition === "duplicate") && !providerFailure.recoverable && failure.code === "provider_exited" + if ((receipt?.disposition === "committed" || receipt?.disposition === "duplicate") && !providerFailure.recoverable && failure.code === "provider_exited" && failure.expected === false && failure.completedByExitedProcess === false && failure.previousTurnCompleted === false && failure.provider === input.provider.kind && event.runId === input.binding.runId && event.normalizedSessionId === session.identity().sessionId) { @@ -1444,7 +1490,7 @@ async function consumeTurn( ? (payload.request as Record) : null; if ( - receipt.disposition === "committed" && + receipt?.disposition === "committed" && event.eventType === "runtime_request.created" && request?.schema === "paperclip.runtime_request.v2" && request.type === "input" && @@ -1555,6 +1601,7 @@ async function consumeTurn( goalStatus(eventGoal) !== "active" && payload.workingNow !== true ) { + await settlePendingDeltas(); return { event, eventCount, @@ -1574,6 +1621,7 @@ async function consumeTurn( latestSessionGoal !== null && goalStatus(latestSessionGoal) !== "active" ) { + await settlePendingDeltas(); return { event, eventCount, @@ -1603,6 +1651,7 @@ async function consumeTurn( if (await supersededTerminal?.(event, appendAbort.signal)) continue; if (providerFailure) throw providerFailure; if (compactionResult && event.eventType === "turn.completed") { + await settlePendingDeltas(); return { event, eventCount, @@ -1612,6 +1661,7 @@ async function consumeTurn( } if (semanticResultProposal === null && governedResult === null && inputTimers.size === 0 && await resumeInterruptedTurn?.(event, appendAbort.signal)) continue; + await settlePendingDeltas(); return { event, eventCount, @@ -1621,13 +1671,17 @@ async function consumeTurn( } } })(); + // The race below sees the consumer's own outcome, so a failure reaches abort + // and provider cancellation at once. Teardown waits, within its bound, for + // the consumer together with its pending delta appends. + const consumer = consumerBody.finally(joinOutstandingDeltas); // A timeout can win the race while an iterator is still waiting for data. // Observe any later consumer rejection so it cannot become process-fatal. void consumer.catch(() => undefined); let consumptionFailed = false; try { return await Promise.race([ - consumer, + consumerBody, new Promise((_, reject) => { if (timeoutMs <= 0) return; // Node timers overflow above ~24.8 days. Keep explicit long deadlines diff --git a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts index b666576f16a..e13cb4fe461 100644 --- a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts +++ b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts @@ -4,7 +4,7 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { eq, inArray, sql } from "drizzle-orm"; import { drizzle } from "drizzle-orm/postgres-js"; import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issues, nativeRunnerCheckpoints } from "@paperclipai/db"; -import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; +import { appendHeartbeatRunEvent, appendHeartbeatRunEventBatch } from "../services/heartbeat-run-events.js"; import { nativeSha256 } from "../services/native-runtime/canonical.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -205,6 +205,79 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { expect((await readEvents(runId)).map(row => row.sourceSeq)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 10, 9]); }); + it("commits a batch of in-order deltas with one lock read, one replay probe and one insert", async () => { + const runId = await makeRun(); + await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); + await appendHeartbeatRunEvent(db, { companyId, agentId, runId, eventType: "stdout", message: "ordinary" }); + const batch = [2, 3, 4, 5, 6].map(sourceSeq => nativeEvent(runId, sourceSeq)); + const appended = await sentStatements(db.$client, () => appendHeartbeatRunEventBatch(db, batch)); + // BEGIN, the run lock, the replay probe, one insert of every row, COMMIT. + expect(appended.statements).toHaveLength(5); + expect(appended.statements.filter(text => text.includes("insert into heartbeat_run_events"))).toHaveLength(1); + expect(appended.result.map(receipt => [receipt.disposition, receipt.row.seq, receipt.row.sourceSeq, receipt.highestContiguousSourceSeq])) + .toEqual([2, 3, 4, 5, 6].map((sourceSeq, index) => ["committed", index + 3, sourceSeq, sourceSeq])); + const rows = await readEvents(runId); + expect(rows.map(row => [row.seq, row.sourceSeq])).toEqual([[1, 1], [2, null], [3, 2], [4, 3], [5, 4], [6, 5], [7, 6]]); + expect(appended.result.map(receipt => receipt.row)).toEqual(rows.slice(2)); + expect(rows[3]).toMatchObject({ eventType: "item.delta", stream: "system", level: "info", payload: { prpEvent: { sourceSeq: 3 } }, + sourceInstanceId: "native-writer", sourceEventId: "native-writer:3", protocolSchemaVersion: 1, + sourcePayloadSha256: nativeSha256({ sourceSeq: 3 }) }); + expect(await readRun(runId)).toMatchObject({ nextEventSeq: 8, nativeSourceCursors: { "native-writer": 6 } }); + // A later single append continues from the batch. + expect(await appendHeartbeatRunEvent(db, nativeEvent(runId, 7))).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 7 }); + }); + + it("replays a batch without new rows and rejects a conflicting replay", async () => { + const runId = await makeRun(); + await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); + const batch = [2, 3, 4].map(sourceSeq => nativeEvent(runId, sourceSeq)); + const first = await appendHeartbeatRunEventBatch(db, batch); + const rows = await readEvents(runId); + const replay = await appendHeartbeatRunEventBatch(db, batch); + expect(replay.map(receipt => receipt.disposition)).toEqual(["duplicate", "duplicate", "duplicate"]); + expect(replay.map(receipt => receipt.row)).toEqual(first.map(receipt => receipt.row)); + expect(replay.every(receipt => receipt.highestContiguousSourceSeq >= 4)).toBe(true); + // A replay window that overlaps the stored end appends only the new events, in order. + const overlap = await appendHeartbeatRunEventBatch(db, [3, 4, 5, 6].map(sourceSeq => nativeEvent(runId, sourceSeq))); + expect(overlap.map(receipt => receipt.disposition)).toEqual(["duplicate", "duplicate", "committed", "committed"]); + expect(overlap.map(receipt => receipt.highestContiguousSourceSeq)).toEqual([4, 4, 5, 6]); + const conflicting = nativeEvent(runId, 7); + await expect(appendHeartbeatRunEventBatch(db, [nativeEvent(runId, 7), { ...nativeEvent(runId, 3), + nativeSource: { ...conflicting.nativeSource, sourceEventId: "native-writer:3", sourceSeq: 3, canonicalPayload: { sourceSeq: 999 } } }])) + .rejects.toThrow("native_event_replay_conflict"); + expect((await readEvents(runId)).map(row => row.sourceSeq)).toEqual([...rows.map(row => row.sourceSeq), 5, 6]); + expect(await readRun(runId)).toMatchObject({ nextEventSeq: 7, nativeSourceCursors: { "native-writer": 6 } }); + }); + + it("stores nothing from a batch that fails before its commit, and commits the resent batch", async () => { + const runId = await makeRun(); + await appendHeartbeatRunEvent(db, nativeEvent(runId, 1)); + const before = await readRun(runId); + const batch = [2, 3, 4].map(sourceSeq => nativeEvent(runId, sourceSeq)); + let checks = 0; + // The writer loses the run after the insert and before the commit. + await expect(appendHeartbeatRunEventBatch(db, batch, () => { if (++checks === 2) throw new Error("writer revoked"); })) + .rejects.toThrow("writer revoked"); + expect(await readRun(runId)).toEqual(before); + expect((await readEvents(runId)).map(row => row.sourceSeq)).toEqual([1]); + const resent = await appendHeartbeatRunEventBatch(db, batch); + expect(resent.map(receipt => [receipt.disposition, receipt.row.seq, receipt.highestContiguousSourceSeq])) + .toEqual([["committed", 2, 2], ["committed", 3, 3], ["committed", 4, 4]]); + }); + + it("appends a batch one by one when its run has no cursor or the batch leaves a gap", async () => { + const runId = await makeRun(); + const fromStart = await appendHeartbeatRunEventBatch(db, [1, 2, 3].map(sourceSeq => nativeEvent(runId, sourceSeq))); + expect(fromStart.map(receipt => receipt.highestContiguousSourceSeq)).toEqual([1, 2, 3]); + const gap = await appendHeartbeatRunEventBatch(db, [5, 6].map(sourceSeq => nativeEvent(runId, sourceSeq))); + expect(gap.map(receipt => [receipt.disposition, receipt.highestContiguousSourceSeq])).toEqual([["committed", 3], ["committed", 3]]); + const filled = await appendHeartbeatRunEventBatch(db, [4, 7].map(sourceSeq => nativeEvent(runId, sourceSeq))); + expect(filled.map(receipt => receipt.highestContiguousSourceSeq)).toEqual([6, 7]); + expect((await readEvents(runId)).map(row => [row.seq, row.sourceSeq])).toEqual([[1, 1], [2, 2], [3, 3], [4, 5], [5, 6], [6, 4], [7, 7]]); + await expect(appendHeartbeatRunEventBatch(db, [nativeEvent(runId, 8), { ...nativeEvent(runId, 9), runId: await makeRun() }])) + .rejects.toThrow("heartbeat_run_event_batch_unsupported"); + }); + it("registers a native event's wakes before its write and only when the event is new", async () => { const runId = await makeRun(); const [issue] = await db.insert(issues).values({ companyId, title: "Progress task" }).returning(); diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index cab99298136..7326396df3f 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -98,22 +98,26 @@ export async function allocateHeartbeatRunEventSeq( return Number(updated.nextEventSeq) - 1; } +function runEventWakes(input: AppendHeartbeatRunEventInput): RunEventWakes { + return { + // Terminal proof may arrive after controller retirement and a quiet Stop + // check. Register the scoped wake before writing it, in the same transaction. + recovery: ["native.process_start_requested", "native.process_identity_recorded", "native.local_process_stopped"].includes(input.eventType) + || Boolean(input.nativeSource && ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(input.eventType)), + reassessment: readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null, + progress: isSafeNativeChatProgressEventType(input.eventType), + response: isNativeResponseEvidence(input.eventType, input.payload?.prpEvent), + retention: input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX)), + }; +} + export async function appendHeartbeatRunEvent( db: Db, input: AppendHeartbeatRunEventInput, ): Promise { - const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX)); - // Terminal proof may arrive after controller retirement and a quiet Stop - // check. Register the scoped wake before writing it, in the same transaction. - const projectsRecovery = ["native.process_start_requested", "native.process_identity_recorded", "native.local_process_stopped"].includes(input.eventType) - || Boolean(input.nativeSource && ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(input.eventType)); - const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null; - const projectsProgress = isSafeNativeChatProgressEventType(input.eventType); - const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent); // Deduplicated receipts return without allocating, so they cannot carry run changes. if (input.runUpdate && (input.nativeSource || input.retryExhaustion)) throw new Error("heartbeat_run_event_run_update_unsupported"); - const wakes = { recovery: projectsRecovery, reassessment: projectsReassessment, progress: projectsProgress, - response: projectsResponse, retention: projectsRetention }; + const wakes = runEventWakes(input); if (!input.runUpdate && !input.nativeSource && !input.retryExhaustion && !input.assertWritable && !Object.values(wakes).some(Boolean)) { const [appended] = await db.execute>(sql` with allocated_run_event as ( @@ -145,6 +149,102 @@ export async function appendHeartbeatRunEvent( }); } +/** + * Appends native source events of one run and one source in one transaction, + * in the given order, and returns one receipt per event. A stream that + * continues its source's stored cursor with new events takes four statements + * for the whole batch: the run lock, a replay probe, one insert of every row, + * and the commit. Any other batch (a replay, a gap, an event that registers a + * wake, or a run without a cursor) appends its events one by one in the same + * transaction, with the checks of a single append. + */ +export async function appendHeartbeatRunEventBatch( + db: Db, + inputs: readonly AppendHeartbeatRunEventInput[], + assertWritable?: () => void, +): Promise { + const first = inputs[0]; + if (!first) return []; + const source = first.nativeSource; + if (!source || inputs.some(input => !input.nativeSource || input.runUpdate || input.retryExhaustion + || input.companyId !== first.companyId || input.runId !== first.runId || input.agentId !== first.agentId + || input.nativeSource.sourceInstanceId !== source.sourceInstanceId)) { + throw new Error("heartbeat_run_event_batch_unsupported"); + } + if (inputs.length === 1) return [await appendHeartbeatRunEvent(db, assertWritable ? { ...first, assertWritable } : first)]; + const wakes = inputs.map(runEventWakes); + return db.transaction(async (transaction) => { + const tx = transaction as unknown as Db; + assertWritable?.(); + const appendOneByOne = async () => { + const receipts: AppendHeartbeatRunEventResult[] = []; + for (const [index, input] of inputs.entries()) { + receipts.push(await appendNativeSourceEvent(tx, input as NativeSourceEventInput, wakes[index]!)); + } + return receipts; + }; + if (wakes.some(wake => Object.values(wake).some(Boolean))) return finish(await appendOneByOne()); + const [run] = await tx.execute<{ company_id: string; agent_id: string; source_cursor: string | number | null }>( + sql`select "company_id", "agent_id", ("native_source_cursors" ->> ${source.sourceInstanceId})::bigint as "source_cursor" + from "heartbeat_runs" where "id" = ${first.runId} for no key update`); + if (!run || run.company_id !== first.companyId || run.agent_id !== first.agentId) { + throw new Error("heartbeat_run_event_binding_mismatch"); + } + const cursor = run.source_cursor === null || run.source_cursor === undefined ? null : Number(run.source_cursor); + const continuesCursor = cursor !== null && inputs.every((input, index) => input.nativeSource!.sourceSeq === cursor + index + 1); + if (!continuesCursor) return finish(await appendOneByOne()); + const eventIds = JSON.stringify(inputs.map(input => input.nativeSource!.sourceEventId)); + const [replayed] = await tx.execute<{ replayed: boolean }>(sql`select exists ( + select 1 from heartbeat_run_events where run_id = ${first.runId} and ( + source_event_id = any(array(select jsonb_array_elements_text(${eventIds}::jsonb))) + or (source_instance_id = ${source.sourceInstanceId} and source_seq > ${cursor}::bigint))) as replayed`); + if (replayed?.replayed) return finish(await appendOneByOne()); + const rows = JSON.stringify(inputs.map(input => ({ + eventType: input.eventType, stream: input.stream ?? null, level: input.level ?? null, color: input.color ?? null, + message: input.message ?? null, + payload: input.payload ? heartbeatRunEvents.payload.mapToDriverValue(input.payload) : null, + sourceEventId: input.nativeSource!.sourceEventId, sourceSeq: input.nativeSource!.sourceSeq, + sourcePayloadSha256: nativeSha256(input.nativeSource!.canonicalPayload), + protocolSchemaVersion: input.nativeSource!.protocolSchemaVersion, + }))); + const lastSeq = cursor + inputs.length; + const inserted = await tx.execute>(sql` + with allocated as ( + update heartbeat_runs + set next_event_seq = next_event_seq + ${inputs.length}::integer, + updated_at = ${sql.param(new Date(), heartbeatRuns.updatedAt)}, + native_source_cursors = jsonb_set(coalesce(native_source_cursors, '{}'::jsonb), + array[${source.sourceInstanceId}::text], to_jsonb(${lastSeq}::bigint)) + where id = ${first.runId} + returning next_event_seq - ${inputs.length}::integer as first_seq + ) + insert into heartbeat_run_events + (company_id, run_id, agent_id, seq, event_type, stream, level, color, message, payload, + source_instance_id, source_event_id, source_seq, source_payload_sha256, protocol_schema_version) + select ${first.companyId}::uuid, ${first.runId}::uuid, ${first.agentId}::uuid, allocated.first_seq + event.ordinal - 1, + event.value ->> 'eventType', event.value ->> 'stream', event.value ->> 'level', event.value ->> 'color', + event.value ->> 'message', (event.value ->> 'payload')::jsonb, + ${source.sourceInstanceId}::text, event.value ->> 'sourceEventId', (event.value ->> 'sourceSeq')::bigint, + event.value ->> 'sourcePayloadSha256', (event.value ->> 'protocolSchemaVersion')::integer + from allocated cross join jsonb_array_elements(${rows}::jsonb) with ordinality as event(value, ordinal) + order by event.ordinal + returning * + `); + if (inserted.length !== inputs.length) throw new Error("heartbeat_run_event_not_persisted"); + const bySourceSeq = new Map(inserted.map(raw => [Number(raw.source_seq), decodeRunEventRow(raw)])); + return finish(inputs.map(input => ({ + row: bySourceSeq.get(input.nativeSource!.sourceSeq)!, + disposition: "committed" as const, + highestContiguousSourceSeq: input.nativeSource!.sourceSeq, + }))); + }); + + function finish(receipts: AppendHeartbeatRunEventResult[]) { + assertWritable?.(); + return receipts; + } +} + const runEventColumns = Object.entries(getTableColumns(heartbeatRunEvents)); /** diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts index a9efea18451..7509476e6f9 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts @@ -904,6 +904,118 @@ describe("PaperclipControlPlanePort conformance", () => { expect(await stored()).toEqual([1, 2, 3, 4].map(seq => ({ seq, sourceSeq: seq }))); }); + const streamingFixture = async (options: ConstructorParameters[2] = {}) => { + const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() }; + const runnerId = randomUUID(); + await db.insert(heartbeatRuns).values({ + id: identity.runId, companyId: identity.companyId, agentId: identity.agentId, + status: "running", runtimeMode: "native", nativeIssueId: identity.issueId, + nativeSessionId: identity.sessionId, runnerInstanceId: runnerId, + completionContractId: contractId, completionContractSha256: contractSha, + contextSnapshot: { issueId: identity.issueId }, + }); + const binding = { ...identity, completionContractId: contractId, completionContractSha256: contractSha, + sourceInstanceId: runnerId, controlPlaneSourceInstanceId: `stream-${identity.runId}` }; + const port = new PaperclipControlPlanePort(db, binding, options); + await port.openRun({ identity, backendKind: "mock", sourceInstanceId: runnerId }); + const emittedAt = new Date().toISOString(); + const event = (sourceSeq: number, eventType: PrpEvent["eventType"] = "item.delta", text = `chunk ${sourceSeq}`): PrpEvent => ({ + schema: "paperclip.prp.event.v1", sourceEventId: `${runnerId}:${sourceSeq}`, sourceSeq, + sourceInstanceId: runnerId, sourceKind: "runner", runId: identity.runId, + normalizedSessionId: identity.sessionId, turnId: "stream-turn", itemId: "stream-message", + eventType, schemaVersion: 1, priority: eventType === "item.delta" ? 1 : 0, emittedAt, + payload: eventType === "item.delta" ? { kind: "agentMessage", text } : { + schema: "paperclip.tool.execution.v1", executionId: `tool-${sourceSeq}`, transport: "builtin", + operation: "execute", name: "diagnostic", target: null, namespace: null, readOnly: true, + status: "running", durationMs: null, exitCode: null, progress: null, + output: null, outputBytes: 0, outputTruncated: false, outputDigest: null, + }, + }); + const streamed = { signal: new AbortController().signal, streaming: true as const }; + const stored = () => db.select({ seq: heartbeatRunEvents.seq, sourceSeq: heartbeatRunEvents.sourceSeq }) + .from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, identity.runId)).orderBy(asc(heartbeatRunEvents.seq)); + return { port, binding, identity, runnerId, event, stored, streamed }; + }; + + it("commits the first text delta at once and later deltas in one batch before the next other event", async () => { + const committed: number[] = []; + // A timer this long would time the test out: only the first delta and the + // next other event may commit the queued deltas. + const f = await streamingFixture({ streamingBatchMs: 600_000, onCommittedEvent: async (event) => { committed.push(event.sourceSeq); } }); + await expect(f.port.appendEvent(f.event(1), f.streamed)).resolves.toEqual({ cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" }); + const options = db.$client.options as { debug?: unknown }; + const statements: string[] = []; + options.debug = (_connection: number, text: string) => { statements.push(text.trim().replace(/\s+/g, " ")); }; + try { + const settled: number[] = []; + const deltas = [2, 3, 4, 5, 6].map(sourceSeq => f.port.appendEvent(f.event(sourceSeq), f.streamed).then(receipt => { settled.push(sourceSeq); return receipt; })); + await new Promise(resolve => setTimeout(resolve, 50)); + // Queued deltas are neither stored nor acknowledged yet. + expect(settled).toEqual([]); + expect(await f.stored()).toEqual([{ seq: 1, sourceSeq: 1 }]); + statements.length = 0; + const boundary = f.port.appendEvent(f.event(7, "tool.execution.started")); + expect(await Promise.all(deltas)).toEqual([2, 3, 4, 5, 6].map(sourceSeq => ({ + cursor: sourceSeq, highestContiguousSourceSeq: sourceSeq, disposition: "committed" }))); + await expect(boundary).resolves.toMatchObject({ cursor: 7, highestContiguousSourceSeq: 7, disposition: "committed" }); + expect(settled).toEqual([2, 3, 4, 5, 6]); + // The five deltas share one transaction and one insert; the boundary event has its own. + expect(statements.filter(text => text.includes("insert into heartbeat_run_events"))).toHaveLength(2); + expect(statements.filter(text => /^begin/i.test(text))).toHaveLength(2); + } finally { options.debug = undefined; } + expect(committed).toEqual([1, 2, 3, 4, 5, 6, 7]); + expect(await f.stored()).toEqual([1, 2, 3, 4, 5, 6, 7].map(seq => ({ seq, sourceSeq: seq }))); + // After another event, the next delta commits at once again. + await expect(f.port.appendEvent(f.event(8), f.streamed)).resolves.toMatchObject({ cursor: 8, disposition: "committed" }); + // A caller that waits for each receipt never waits for the timer. + await expect(f.port.appendEvent(f.event(9))).resolves.toMatchObject({ cursor: 9, disposition: "committed" }); + // A delta without text does not end the immediate commits: the first text still commits at once. + await f.port.appendEvent(f.event(10, "tool.execution.started")); + await expect(f.port.appendEvent(f.event(11, "item.delta", ""), f.streamed)).resolves.toMatchObject({ cursor: 11 }); + await expect(f.port.appendEvent(f.event(12), f.streamed)).resolves.toMatchObject({ cursor: 12 }); + }); + + it("observes every committed delta of a batch when one observer fails", async () => { + const observed: number[] = []; + const f = await streamingFixture({ streamingBatchMs: 600_000, onCommittedEvent: async (event) => { + if (event.sourceSeq === 3) throw new Error("log write failed"); + observed.push(event.sourceSeq); + } }); + await f.port.appendEvent(f.event(1), f.streamed); + const receipts = [2, 3, 4].map(sourceSeq => f.port.appendEvent(f.event(sourceSeq), f.streamed)); + f.port.flushStreamingEvents(); + await expect(receipts[0]).resolves.toMatchObject({ cursor: 2 }); + await expect(receipts[1]).rejects.toThrow("log write failed"); + await expect(receipts[2]).resolves.toMatchObject({ cursor: 4 }); + expect(observed).toEqual([1, 2, 4]); + expect(await f.stored()).toEqual([1, 2, 3, 4].map(seq => ({ seq, sourceSeq: seq }))); + }); + + it("flushes queued deltas on the timer and before a replay read, and rejects every delta of a failed batch", async () => { + let active = true; + const f = await streamingFixture({ streamingBatchMs: 20, assertControllerActive: () => { if (!active) throw new Error("controller_revoked"); } }); + await f.port.appendEvent(f.event(1), f.streamed); + // Later deltas commit on the timer when no other event follows. + expect((await Promise.all([2, 3].map(sourceSeq => f.port.appendEvent(f.event(sourceSeq), f.streamed)))).map(receipt => receipt.cursor)).toEqual([2, 3]); + // Replay reads include queued deltas. + const queued = f.port.appendEvent(f.event(4), f.streamed); + const replay = await f.port.replayEvents({ runId: f.identity.runId, sourceInstanceId: f.runnerId, afterSourceSeq: 0, limit: 10 }); + expect(replay.highestContiguousSourceSeq).toBe(4); + await expect(queued).resolves.toMatchObject({ cursor: 4 }); + // The controller loses the run before the batch commits: every queued delta + // fails and nothing is stored, so the runner's resend commits them. + const failing = [5, 6, 7].map(sourceSeq => f.port.appendEvent(f.event(sourceSeq), f.streamed)); + active = false; + for (const receipt of failing) await expect(receipt).rejects.toThrow("controller_revoked"); + active = true; + expect((await f.stored()).map(row => row.sourceSeq)).toEqual([1, 2, 3, 4]); + const replacement = new PaperclipControlPlanePort(db, f.binding, { streamingBatchMs: 20 }); + await replacement.openRun({ identity: f.identity, backendKind: "mock", sourceInstanceId: f.runnerId }); + const resent = await Promise.all([4, 5, 6, 7].map(sourceSeq => replacement.appendEvent(f.event(sourceSeq), f.streamed))); + expect(resent.map(receipt => [receipt.cursor, receipt.disposition])).toEqual([[4, "duplicate"], [5, "committed"], [6, "committed"], [7, "committed"]]); + expect(await f.stored()).toEqual([1, 2, 3, 4, 5, 6, 7].map(seq => ({ seq, sourceSeq: seq }))); + }); + it("omits API credentials from persisted tool events and checkpoints without changing wire results", async () => { const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, runId: randomUUID(), sessionId: randomUUID() }; const runnerId = randomUUID(); diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.ts b/server/src/services/native-runtime/paperclip-control-plane-port.ts index f09d48d48b9..679bbf4cb2a 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.ts @@ -28,7 +28,8 @@ import { validatePrpEvent, validatePrpStructuredRunResult, } from "../../vendor/paperclip-runner/index.js"; -import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; +import { appendHeartbeatRunEvent, appendHeartbeatRunEventBatch, type AppendHeartbeatRunEventInput, + type AppendHeartbeatRunEventResult } from "../heartbeat-run-events.js"; import { publishLiveEvent } from "../live-events.js"; import { nativeSha256 } from "./canonical.js"; import { redactRunnerApiEvent, redactRunnerApiPayloads } from "./runner-api-log-redaction.js"; @@ -53,6 +54,21 @@ export interface PaperclipControlPlaneBinding { const IDENTITY_DELTA_RETRY_WINDOW = 1_024; const IDENTITY_DELTA_RETRY_BYTES = 4 * 1024 * 1024; +/** Text deltas that follow the first one after another event wait at most this long for their batch. */ +export const STREAMING_EVENT_BATCH_MS = 250; +/** A batch that reaches this many deltas commits at once. */ +export const STREAMING_EVENT_BATCH_MAX = 100; + +type AppendReceipt = { cursor: number; highestContiguousSourceSeq: number; disposition: "committed" | "duplicate" }; +type PendingStreamingEvent = { + event: PrpEvent; + input: AppendHeartbeatRunEventInput; + options?: { signal: AbortSignal }; + startedAtMs: number; + startedClock: number; + resolve: (receipt: AppendReceipt) => void; + reject: (error: unknown) => void; +}; function isPrpEvent(value: NativeRunEvent | PrpEvent): value is PrpEvent { return "schema" in value && [ @@ -103,6 +119,23 @@ export class PaperclipControlPlanePort implements ControlPlanePort { readonly #taskSessionPublicationOwner?: NativeRunnerCheckpointOwner; readonly #deferredEventObservers = new Map>(); #deferredObserverFailure: { error: unknown } | null = null; + /** + * Text deltas commit in batches. A caller may start the next append before + * a delta's receipt settles; the receipt settles after its batch commits. + */ + readonly pipelinesStreamingEvents = true; + readonly #streamingBatchMs: number; + #streamingBatch: PendingStreamingEvent[] = []; + #streamingTimer: ReturnType | null = null; + #streamingTail: Promise = Promise.resolve(); + /** Deltas queued or committing. Other operations wait only when some are; otherwise they start at once. */ + #streamingEventsPending = 0; + /** + * False until a delta with text commits after the latest other event. Until + * then each delta commits at once, so the first text of an item, and the + * time to first token, never wait for the batch timer. + */ + #streamedSinceBoundary = false; constructor( db: Db, @@ -121,8 +154,11 @@ export class PaperclipControlPlanePort implements ControlPlanePort { nativeRecovery?: { compatibility: NativeControllerCompatibility; owner: NativeRunnerCheckpointOwner }; taskSessionPublication?: NativeTaskSessionPublicationContext; taskSessionPublicationOwner?: NativeRunnerCheckpointOwner; + /** How long a text delta that is not the first after another event may wait for its batch. */ + streamingBatchMs?: number; } = {}, ) { + this.#streamingBatchMs = options.streamingBatchMs ?? STREAMING_EVENT_BATCH_MS; this.#db = db; this.#assertControllerActive = options.assertControllerActive; this.#assertRunAdmissionActive = options.assertRunAdmissionActive ?? options.assertControllerActive; @@ -239,6 +275,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort { async checkpointSession(snapshot: PersistedNativeSession, options?: { signal: AbortSignal }): Promise { this.#assertActive(options); + if (this.#streamingEventsPending > 0) await this.#flushStreamingEvents(); if (this.#deferredEventObservers.size > 0 || this.#deferredObserverFailure) await this.#settleDeferredObservers(options); snapshot = redactRunnerApiPayloads(this.#identityRedactor.redact(snapshot)); const identity = snapshot.identity; @@ -357,7 +394,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort { }); } - async appendEvent(value: NativeRunEvent | PrpEvent, options?: { signal: AbortSignal; deferPostCommitObservers?: true }) { + async appendEvent(value: NativeRunEvent | PrpEvent, options?: { signal: AbortSignal; deferPostCommitObservers?: true; streaming?: true }) { const persistenceStartedAt = Date.now(), persistenceClock = performance.now(); this.#assertActive(options); if (!isPrpEvent(value)) throw new Error("native_legacy_event_not_supported"); @@ -410,8 +447,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort { } event = redactRunnerApiEvent(event); const observerKey = JSON.stringify([event.sourceInstanceId, event.sourceEventId]); - const pendingObservation = this.#deferredEventObservers.get(observerKey); - const persisted = await appendHeartbeatRunEvent(this.#db, { + const input: AppendHeartbeatRunEventInput = { companyId: this.#binding.companyId, runId: this.#binding.runId, agentId: this.#binding.agentId, @@ -429,39 +465,21 @@ export class PaperclipControlPlanePort implements ControlPlanePort { // The append has no publication side effects. Revocation rolls back // its event and sequence allocation before the append commits. assertWritable: () => this.#assertActive(options), - }); + }; + if (event.eventType === "item.delta" && !options?.deferPostCommitObservers) { + return this.#appendStreamingEvent({ event, input, options, startedAtMs: persistenceStartedAt, startedClock: persistenceClock }, + options?.streaming === true); + } + // Every other event commits after each delta appended before it. + if (this.#streamingEventsPending > 0) await this.#flushStreamingEvents(); + this.#streamedSinceBoundary = false; + this.#assertActive(options); + const pendingObservation = this.#deferredEventObservers.get(observerKey); + const persisted = await appendHeartbeatRunEvent(this.#db, input); this.#assertActive(options); const persistenceEndedAt = persistenceStartedAt + performance.now() - persistenceClock; let observation = pendingObservation ?? this.#deferredEventObservers.get(observerKey); - if (!observation) observation = (async () => { - try { await this.#onEventPersistence?.(event, persistenceStartedAt, persistenceEndedAt); } catch {} - if (persisted.disposition === "committed") { - try { - publishLiveEvent({ - companyId: this.#binding.companyId, - type: "heartbeat.run.event", - payload: { - runId: this.#binding.runId, - agentId: this.#binding.agentId, - issueId: this.#binding.issueId, - seq: persisted.row.seq, - eventType: persisted.row.eventType, - stream: persisted.row.stream, - level: persisted.row.level, - payload: persisted.row.payload, - }, - }); - } catch { - } - await this.#onCommittedEvent?.(event, persisted.row.seq); - } else { - // A recovered runner may replay the event whose durable side effects - // parked the prior attempt. Do not repeat those effects, but let the - // embedding runtime refresh observational state before appendEvent - // returns to its synchronous governed-wait boundary. - await this.#onDuplicateEvent?.(event, persisted.row.seq); - } - })(); + if (!observation) observation = this.#observePersistedEvent(event, persisted, persistenceStartedAt, persistenceEndedAt); if (options?.deferPostCommitObservers) { this.#deferredEventObservers.set(observerKey, observation); void observation.then(() => { @@ -480,6 +498,108 @@ export class PaperclipControlPlanePort implements ControlPlanePort { }; } + async #observePersistedEvent( + event: PrpEvent, + persisted: AppendHeartbeatRunEventResult, + persistenceStartedAt: number, + persistenceEndedAt: number, + ): Promise { + try { await this.#onEventPersistence?.(event, persistenceStartedAt, persistenceEndedAt); } catch {} + if (persisted.disposition === "committed") { + try { + publishLiveEvent({ + companyId: this.#binding.companyId, + type: "heartbeat.run.event", + payload: { + runId: this.#binding.runId, + agentId: this.#binding.agentId, + issueId: this.#binding.issueId, + seq: persisted.row.seq, + eventType: persisted.row.eventType, + stream: persisted.row.stream, + level: persisted.row.level, + payload: persisted.row.payload, + }, + }); + } catch { + } + await this.#onCommittedEvent?.(event, persisted.row.seq); + } else { + // A recovered runner may replay the event whose durable side effects + // parked the prior attempt. Do not repeat those effects, but let the + // embedding runtime refresh observational state before appendEvent + // returns to its synchronous governed-wait boundary. + await this.#onDuplicateEvent?.(event, persisted.row.seq); + } + } + + /** + * Queues a text delta for the next batch. A caller that waits for each + * receipt (no `streaming`) commits at once. Deltas up to the first one with + * text after any other event also commit at once. + * Later streaming deltas wait at most the batch time, or until the batch is full. + */ + #appendStreamingEvent(pending: Omit, streaming: boolean): Promise { + return new Promise((resolve, reject) => { + this.#streamingBatch.push({ ...pending, resolve, reject }); + this.#streamingEventsPending += 1; + if (!streaming || !this.#streamedSinceBoundary || this.#streamingBatch.length >= STREAMING_EVENT_BATCH_MAX) { + void this.#flushStreamingEvents(); + } else { + this.#streamingTimer ??= setTimeout(() => { void this.#flushStreamingEvents(); }, this.#streamingBatchMs); + } + }); + } + + flushStreamingEvents(): void { + void this.#flushStreamingEvents(); + } + + /** Commits the queued deltas. Resolves after every earlier batch has settled, and never rejects. */ + #flushStreamingEvents(): Promise { + if (this.#streamingTimer !== null) clearTimeout(this.#streamingTimer); + this.#streamingTimer = null; + const batch = this.#streamingBatch; + if (batch.length > 0) { + this.#streamingBatch = []; + if (batch.some(pending => typeof pending.event.payload.text === "string" && pending.event.payload.text.length > 0)) { + this.#streamedSinceBoundary = true; + } + this.#streamingTail = this.#streamingTail.then(() => this.#commitStreamingEvents(batch)) + .finally(() => { this.#streamingEventsPending -= batch.length; }); + } + return this.#streamingTail; + } + + /** + * One transaction for the batch. Each committed row is then observed in + * order; an observer failure rejects only that delta's receipt, and the + * later rows are still observed. + */ + async #commitStreamingEvents(batch: PendingStreamingEvent[]): Promise { + let receipts: AppendHeartbeatRunEventResult[]; + try { + const assertWritable = () => { for (const pending of batch) this.#assertActive(pending.options); }; + receipts = await appendHeartbeatRunEventBatch(this.#db, batch.map(pending => pending.input), assertWritable); + assertWritable(); + } catch (error) { + for (const pending of batch) pending.reject(error); + return; + } + for (const [index, pending] of batch.entries()) { + const persisted = receipts[index]!; + const endedAt = pending.startedAtMs + performance.now() - pending.startedClock; + try { + await this.#observePersistedEvent(pending.event, persisted, pending.startedAtMs, endedAt); + } catch (error) { + pending.reject(error); + continue; + } + pending.resolve({ cursor: persisted.row.seq, highestContiguousSourceSeq: persisted.highestContiguousSourceSeq, + disposition: persisted.disposition }); + } + } + async replayEvents(input: ReplayControlPlaneEventsInput) { if ( input.runId !== this.#binding.runId @@ -487,6 +607,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort { ) { throw new Error("native_replay_binding_mismatch"); } + if (this.#streamingEventsPending > 0) await this.#flushStreamingEvents(); const rows = await this.#db .select({ payload: heartbeatRunEvents.payload, sourceSeq: heartbeatRunEvents.sourceSeq }) .from(heartbeatRunEvents) @@ -517,6 +638,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort { */ async provesNoRunProviderWork(options: { signal: AbortSignal }): Promise { options.signal.throwIfAborted(); + if (this.#streamingEventsPending > 0) await this.#flushStreamingEvents(); const { companyId, runId } = this.#binding; const providerEvents = sql.join([...NATIVE_PROVIDER_DURABLE_EVENT_TYPES].map(value => sql`${value}`), sql`, `); const result = await this.#db.execute(sql`select @@ -540,6 +662,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort { async completeRun(value: NativeRunResult | CompleteControlPlaneRunInput, options?: { signal: AbortSignal }): Promise { this.#assertActive(options); + if (this.#streamingEventsPending > 0) await this.#flushStreamingEvents(); if (this.#deferredEventObservers.size > 0 || this.#deferredObserverFailure) await this.#settleDeferredObservers(options); value = redactRunnerApiPayloads(this.#identityRedactor.redact(value)); if (!isCompleteInput(value)) throw new Error("native_structured_result_required");