diff --git a/packages/db/src/work-signals.test.ts b/packages/db/src/work-signals.test.ts index 704afac48a7..b4d652eec08 100644 --- a/packages/db/src/work-signals.test.ts +++ b/packages/db/src/work-signals.test.ts @@ -3,7 +3,7 @@ import { drizzle } from "drizzle-orm/postgres-js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { createDb, withDedicatedDbConnection } from "./client.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; -import { DATABASE_WORK_CHANNEL, databaseWorkScopeKey, databaseWorkPending, enableDatabaseWorkNotifications, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, primeDatabaseWorkTransactionId, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; +import { DATABASE_WORK_CHANNEL, databaseWorkScopeKey, databaseWorkPending, enableDatabaseWorkNotifications, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, primeDatabaseWorkTransactionId, publishDatabaseWorkSettled, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; const support = await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("database work signals", () => { @@ -161,6 +161,33 @@ const support = await getEmbeddedPostgresTestSupport(); expect(await db.execute(sql`select * from work_signal_test where id = 'primed-disconnect'`)).toHaveLength(0); }); + it("publishes a settled wake at once outside a transaction, and inside one only after the outermost commit", async () => { + const events: Array<[string, string[]]> = []; + const unsubscribe = subscribeDatabaseWork(db, "settled-after-commit", (event, hints) => events.push([event, [...hints ?? []]])); + try { + publishDatabaseWorkSettled(db, "settled-after-commit", "outside"); + expect(events).toEqual([["settled", ["outside"]]]); + events.length = 0; + await db.transaction(async tx => { + publishDatabaseWorkSettled(tx, "settled-after-commit", "committed"); + await tx.transaction(async savepoint => { publishDatabaseWorkSettled(savepoint, "settled-after-commit", "released"); }); + await expect(tx.transaction(async savepoint => { + publishDatabaseWorkSettled(savepoint, "settled-after-commit", "rolled-back-savepoint"); + throw new Error("savepoint rollback"); + })).rejects.toThrow("savepoint rollback"); + expect(events).toEqual([]); + }); + expect(events).toEqual([["settled", ["committed"]], ["settled", ["released"]]]); + events.length = 0; + await expect(db.transaction(async tx => { + publishDatabaseWorkSettled(tx, "settled-after-commit", "rolled-back"); + throw new Error("outer rollback"); + })).rejects.toThrow("outer rollback"); + expect(events).toEqual([]); + expect(databaseWorkPending(db, "settled-after-commit")).toBe(false); + } finally { unsubscribe(); } + }); + it("recovers after a real backend disconnect without replaying the queue write", async () => { await expect(db.transaction(async tx => { await signalDatabaseWork(tx, "disconnect"); diff --git a/packages/db/src/work-signals.ts b/packages/db/src/work-signals.ts index aa312072cd7..6b4bdbd126a 100644 --- a/packages/db/src/work-signals.ts +++ b/packages/db/src/work-signals.ts @@ -7,7 +7,7 @@ import { randomUUID } from "node:crypto"; type Listener = (event: "intent" | "settled", hints?: ReadonlySet) => void; type Transaction = { topics: Set; hints: Map>; xid?: Promise; uncertain: boolean }; type Scope = { key: object; listeners: Map>; transactions: Set; root: Connection; notificationOrigin: string; notify: boolean }; -type Context = { scope: Scope; transaction?: Transaction; pending?: Map; ended?: boolean }; +type Context = { scope: Scope; transaction?: Transaction; pending?: Map; settled?: Array<[string, string | undefined]>; ended?: boolean }; type Connection = { transaction: (...args: any[]) => Promise; execute: (...args: any[]) => Promise }; const contexts = new WeakMap(); @@ -35,7 +35,7 @@ function install(connection: Connection, context: Context) { try { let level: Context | undefined; const result = await transaction.call(this, async (tx: Connection) => { - const current: Context = { scope: context.scope, transaction: state, pending: new Map() }; + const current: Context = { scope: context.scope, transaction: state, pending: new Map(), settled: [] }; level = current; install(tx, current); try { @@ -48,6 +48,8 @@ function install(connection: Connection, context: Context) { if (context.pending && level?.pending) { for (const [key, payload] of level.pending) if (!context.pending.has(key)) context.pending.set(key, payload); } + if (context.transaction) context.settled?.push(...level?.settled ?? []); + else for (const [topic, hint] of level?.settled ?? []) publish(context.scope, topic, "settled", hint === undefined ? undefined : new Set([hint])); if (!context.transaction) context.scope.transactions.delete(state); return result; } catch (error) { @@ -128,6 +130,23 @@ export async function notifyDatabaseWorkOnCommit(connection: object, topic: stri notifyAtCommit(context, notification(context.scope, topic)); } +export function databaseWorkCommitNotification(owner: object, topic: string): string | null { + const scope = contexts.get(owner)?.scope; + return scope?.notify ? notification(scope, topic) : null; +} + +export function publishDatabaseWorkSettled(owner: object, topic: string, hint?: string): void { + const context = contexts.get(owner); + if (!context) return; + if (!context.transaction) { + publish(context.scope, topic, "settled", hint === undefined ? undefined : new Set([hint])); + return; + } + if (context.ended) throw new Error("Database work was signalled after its transaction or savepoint ended"); + context.settled!.push([topic, hint]); + if (context.scope.notify) notifyAtCommit(context, notification(context.scope, topic)); +} + /** A post-commit invalidation hint, for already-persisted live UI state. */ export async function publishDatabaseWorkHint(owner: object, topic: string): Promise { const scope = contexts.get(owner)?.scope; diff --git a/server/src/__tests__/chat-native-progress-work.test.ts b/server/src/__tests__/chat-native-progress-work.test.ts index d2c981459ec..e1a203e93a0 100644 --- a/server/src/__tests__/chat-native-progress-work.test.ts +++ b/server/src/__tests__/chat-native-progress-work.test.ts @@ -239,6 +239,60 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu } }); + const progressEvent = (sourceSeq: number) => ({ companyId: f.companyId, agentId: f.agentId, runId, + eventType: "tool.execution.progressed", message: "PRIVATE event text", payload: { token: "PRIVATE token" }, + nativeSource: { sourceInstanceId: runId, sourceEventId: `event-${sourceSeq}`, sourceSeq, protocolSchemaVersion: 1, + canonicalPayload: { eventType: "tool.execution.progressed", sourceSeq } } }); + const progressPublished = async () => { + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(1000); + expect(await publications()).toHaveLength(1); + }, { timeout: 5000 }); + }; + const appendWithoutWake = async (sourceSeq: number) => { + const [run] = await db.select({ nextEventSeq: heartbeatRuns.nextEventSeq }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, agentId: f.agentId, runId, seq: run!.nextEventSeq, + eventType: "tool.execution.progressed", message: "PRIVATE event text", payload: { token: "PRIVATE token" }, + sourceInstanceId: runId, sourceEventId: `event-${sourceSeq}`, sourceSeq, protocolSchemaVersion: 1, sourcePayloadSha256: "no-wake" }); + await db.update(heartbeatRuns).set({ nextEventSeq: run!.nextEventSeq + 1, nativeSourceCursors: { [runId]: sourceSeq } }) + .where(eq(heartbeatRuns.id, runId)); + }; + + it("wakes the projection from a one-statement progress event after the statement commits", async () => { + await start(); await settle(); await quiet(); expect(await publications()).toEqual([]); + const signals: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, kind => signals.push(kind)); + const statements: string[] = []; + const options = db.$client.options as { debug?: unknown }; + options.debug = (_connection: number, text: string) => { statements.push(text.trim().replace(/\s+/g, " ")); }; + try { + expect((await appendHeartbeatRunEvent(db, progressEvent(1))).disposition).toBe("committed"); + } finally { options.debug = undefined; unsubscribe(); } + expect(statements).toHaveLength(1); + expect(statements[0]).toMatch(/^with snapshot_run as .* pg_notify/); + expect(signals).toEqual(["settled"]); + await progressPublished(); + await settle(); await quiet(); expect(errors).toEqual([]); + }); + + it("repairs a lost progress wake with the run's next progress event", async () => { + await start(); await settle(); await quiet(); + await appendWithoutWake(1); + await settle(); await quiet(); + expect(await publications()).toEqual([]); + await appendHeartbeatRunEvent(db, progressEvent(2)); + await progressPublished(); + await settle(); await quiet(); expect(errors).toEqual([]); + }); + + it("repairs a lost progress wake with the startup reconcile", async () => { + await appendWithoutWake(1); + expect(await publications()).toEqual([]); + await start(); + await progressPublished(); + await settle(); await quiet(); expect(errors).toEqual([]); + }); + it("wakes a saved event when its first working publication commits after event selection", async () => { await db.delete(chatPublications).where(eq(chatPublications.issueId, f.issueId)); await appendHeartbeatRunEvent(db, event(true)); diff --git a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts index 9ff9edb5694..71433c1cb0f 100644 --- a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts +++ b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts @@ -5,6 +5,7 @@ import { eq, inArray, sql } from "drizzle-orm"; import { drizzle } from "drizzle-orm/postgres-js"; import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issues, nativeRunnerCheckpoints, databaseWorkPending, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; +import { DATABASE_WORK_CHANNEL, enableDatabaseWorkNotifications } from "@paperclipai/db/work-signals"; import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { appendHeartbeatRunEvent, appendHeartbeatRunEventBatch } from "../services/heartbeat-run-events.js"; import { nativeSha256 } from "../services/native-runtime/canonical.js"; @@ -484,30 +485,135 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { .rejects.toThrow("heartbeat_run_event_batch_unsupported"); }); - it("registers a native event's wakes before its write and only when the event is new", async () => { + const finalMessage = (runId: string, sourceSeq: number) => { + const event = nativeEvent(runId, sourceSeq, "item.completed"); + const prpEvent = { sourceSeq, payload: { kind: "agentMessage", channel: "final", text: "Done" } }; + return { ...event, payload: { prpEvent }, nativeSource: { ...event.nativeSource, canonicalPayload: prpEvent } }; + }; + const nativeTaskRun = async (title: string) => { const runId = await makeRun(); - const [issue] = await db.insert(issues).values({ companyId, title: "Progress task" }).returning(); + const [issue] = await db.insert(issues).values({ companyId, title }).returning(); await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId)); + return runId; + }; + + it("appends an event whose only wake is chat progress in one statement and wakes progress after it commits", async () => { + const runId = await nativeTaskRun("Progress task"); const wakes: string[] = []; const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, event => { wakes.push(event); }); try { - const first = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed"))); + const first = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "tool.execution.progressed"))); + expect(first.statements).toHaveLength(1); + expect(first.statements[0]).toMatch(/^with snapshot_run as .* insert into heartbeat_run_events .* pg_notify/); + expect(wakes).toEqual(["settled"]); + expect(first.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 1 }); + wakes.length = 0; + const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "tool.execution.progressed"))); + expect(wakes).toEqual([]); + expect(replay.result).toEqual({ row: first.result.row, disposition: "duplicate", highestContiguousSourceSeq: 1 }); + expect(await readEvents(runId)).toEqual([first.result.row]); + } finally { unsubscribe(); } + }); + + it("keeps a progress-only event inside a caller's transaction on the locked path, and settles its wake only when that transaction ends", async () => { + const runId = await nativeTaskRun("Progress in a caller transaction"); + const wakes: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, event => { wakes.push(event); }); + try { + const committed = await sentStatements(db.$client, () => db.transaction(async tx => { + await appendHeartbeatRunEvent(tx as unknown as typeof db, nativeEvent(runId, 1, "tool.execution.progressed")); + expect(wakes).not.toContain("settled"); + })); + expect(committed.statements[0]).toMatch(/^begin/i); + expect(committed.statements.some(text => text.startsWith("with snapshot_run as"))).toBe(false); + expect(wakes).toEqual(["intent", "settled"]); + wakes.length = 0; + await expect(db.transaction(async tx => { + await appendHeartbeatRunEvent(tx as unknown as typeof db, nativeEvent(runId, 2, "tool.execution.progressed")); + expect(wakes).not.toContain("settled"); + throw new Error("caller rollback"); + })).rejects.toThrow("caller rollback"); + expect((await readEvents(runId)).map(row => row.sourceSeq)).toEqual([1]); + await reconcileDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress); + expect(databaseWorkPending(db, DELIVERY_QUEUES.chatNativeProgress)).toBe(false); + } finally { unsubscribe(); } + }); + + it("sends the progress wake to other server processes with the one statement's commit", async () => { + const runId = await nativeTaskRun("Cross-process progress"); + const notifying = createDb(externalTestDatabaseUrl ?? temporary!.connectionString); + enableDatabaseWorkNotifications(notifying); + const received: string[] = []; + const listener = await otherDb.$client.listen(DATABASE_WORK_CHANNEL, payload => { received.push(payload); }); + try { + const appended = await sentStatements(notifying.$client, () => + appendHeartbeatRunEvent(notifying, nativeEvent(runId, 1, "tool.execution.completed"))); + expect(appended.statements.filter(text => !text.includes("pg_catalog"))).toHaveLength(1); + await vi.waitFor(() => { + expect(received.map(payload => JSON.parse(payload).topic)).toEqual([DELIVERY_QUEUES.chatNativeProgress]); + }, { timeout: 5_000 }); + await appendHeartbeatRunEvent(notifying, nativeEvent(runId, 1, "tool.execution.completed")); + await new Promise(resolve => setTimeout(resolve, 200)); + expect(received).toHaveLength(1); + } finally { + await listener.unlisten(); + await notifying.$client.end({ timeout: 1 }); + } + }); + + it("registers a response wake with chat progress before its write and only when the event is new", async () => { + const runId = await nativeTaskRun("Response task"); + const wakes: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, event => { wakes.push(event); }); + try { + const first = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, finalMessage(runId, 1))); expect(first.statements).toHaveLength(4); expect(first.statements[0]).toMatch(/^begin/i); expect(first.statements[1]).toMatch(/^with "snapshot_run" as .*pg_current_xact_id\(\).* for no key update.*"replay_found"/); expect(first.statements[2]).toMatch(/insert into heartbeat_run_events/); expect(first.statements[3]).toMatch(/^commit/i); expect(wakes).toEqual(["intent", "settled"]); - expect(first.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 1 }); wakes.length = 0; - const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed"))); + const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, finalMessage(runId, 1))); expect(wakes).toEqual([]); expect(replay.statements).toHaveLength(4); expect(replay.result).toEqual({ row: first.result.row, disposition: "duplicate", highestContiguousSourceSeq: 1 }); - expect(await readEvents(runId)).toEqual([first.result.row]); } finally { unsubscribe(); } }); + it("keeps the transaction for the last events of a turn and a run, and one statement for progress-only events", async () => { + const runId = await nativeTaskRun("Last events"); + const events = [ + ["tool.execution.started", nativeEvent(runId, 1, "tool.execution.started")], + ["tool.execution.progressed", nativeEvent(runId, 2, "tool.execution.progressed")], + ["tool.execution.completed", nativeEvent(runId, 3, "tool.execution.completed")], + ["item.completed (not a final message)", nativeEvent(runId, 4, "item.completed")], + ["item.completed (final message)", finalMessage(runId, 5)], + ["run.result.accepted", nativeEvent(runId, 6, "run.result.accepted")], + ["turn.completed", nativeEvent(runId, 7, "turn.completed")], + ["turn.failed", nativeEvent(runId, 8, "turn.failed")], + ["turn.cancelled", nativeEvent(runId, 9, "turn.cancelled")], + ["turn.interrupted", nativeEvent(runId, 10, "turn.interrupted")], + ] as const; + const paths: Array<[string, string]> = []; + for (const [name, input] of events) { + const { statements } = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, input)); + paths.push([name, /^begin/i.test(statements[0] ?? "") ? "transaction" : "one statement"]); + } + expect(paths).toEqual([ + ["tool.execution.started", "one statement"], + ["tool.execution.progressed", "one statement"], + ["tool.execution.completed", "one statement"], + ["item.completed (not a final message)", "one statement"], + ["item.completed (final message)", "transaction"], + ["run.result.accepted", "transaction"], + ["turn.completed", "transaction"], + ["turn.failed", "transaction"], + ["turn.cancelled", "transaction"], + ["turn.interrupted", "transaction"], + ]); + }); + it("gives the wake the transaction id of the transaction that took the run lock", async () => { const runId = await makeRun(); const [issue] = await db.insert(issues).values({ companyId, title: "Primed wake" }).returning(); @@ -591,7 +697,7 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { const runId = await makeRun(); const [issue] = await db.insert(issues).values({ companyId, title: "Progress race" }).returning(); await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId)); - const event = nativeEvent(runId, 1, "item.completed"); + const event = finalMessage(runId, 1); let release!: () => void, locked!: () => void; const released = new Promise(resolve => { release = resolve; }); const lockedRun = new Promise(resolve => { locked = resolve; }); diff --git a/server/src/services/chat-native-progress-notifications.ts b/server/src/services/chat-native-progress-notifications.ts index cc56fd9eac7..8bb7fdf50c7 100644 --- a/server/src/services/chat-native-progress-notifications.ts +++ b/server/src/services/chat-native-progress-notifications.ts @@ -1,6 +1,15 @@ +import { databaseWorkCommitNotification, publishDatabaseWorkSettled } from "@paperclipai/db/work-signals"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import type { ChatRunProjectionScope } from "./chat-run-publication-policy.js"; export function notifyChatNativeProgress(tx: object, scope: ChatRunProjectionScope) { return notifyDeliveryWork(tx, DELIVERY_QUEUES.chatNativeProgress, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId })); } + +export function chatNativeProgressCommitNotification(owner: object): string | null { + return databaseWorkCommitNotification(owner, DELIVERY_QUEUES.chatNativeProgress); +} + +export function publishChatNativeProgressSettled(owner: object, scope: ChatRunProjectionScope): void { + publishDatabaseWorkSettled(owner, DELIVERY_QUEUES.chatNativeProgress, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId })); +} diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index 30f41f5c78a..dc355a9a960 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -4,12 +4,13 @@ import { readNativeEventEvidenceOutcome } from "./native-runtime/evidence-classi import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js"; import { isSafeNativeChatProgressEventType } from "./safe-native-chat-progress.js"; -import { notifyChatNativeProgress } from "./chat-native-progress-notifications.js"; +import { chatNativeProgressCommitNotification, notifyChatNativeProgress, publishChatNativeProgressSettled } from "./chat-native-progress-notifications.js"; import { boundRunIssueIdSql } from "./run-task-binding.js"; import { and, asc, eq, getTableColumns, is, sql } from "drizzle-orm"; -import { PgTransaction } from "drizzle-orm/pg-core"; +import { PgDialect, PgTransaction } from "drizzle-orm/pg-core"; import type { Db } from "@paperclipai/db"; import { heartbeatRunEvents, heartbeatRuns, primeDatabaseWorkTransactionId } from "@paperclipai/db"; +import { DATABASE_WORK_CHANNEL } from "@paperclipai/db/work-signals"; import { nativeSha256 } from "./native-runtime/canonical.js"; import { isUniqueViolation } from "../db-errors.js"; @@ -142,9 +143,11 @@ export async function appendHeartbeatRunEvent( if (!appended) throw new Error("heartbeat_run_event_binding_mismatch"); return { row: decodeRunEventRow(appended), disposition: "committed", highestContiguousSourceSeq: 0 }; } - if (input.nativeSource && !input.runUpdate && !input.retryExhaustion && !Object.values(wakes).some(Boolean) && !is(db, PgTransaction)) { + const progressOnly = wakes.progress && !wakes.recovery && !wakes.reassessment && !wakes.response && !wakes.retention; + if (input.nativeSource && !input.runUpdate && !input.retryExhaustion && (progressOnly || !Object.values(wakes).some(Boolean)) + && !is(db, PgTransaction)) { input.assertWritable?.(); - const receipts = await appendNewNativeSourceEvents(db, [input as NativeSourceEventInput]); + const receipts = await appendNewNativeSourceEvents(db, [input as NativeSourceEventInput], progressOnly); if (receipts) { input.assertWritable?.(); return receipts[0]!; @@ -289,9 +292,52 @@ const APPEND_NEW_NATIVE_SOURCE_EVENTS_SQL = ` order by event.ordinal returning *`; +const BOUND_RUN_ISSUE_ID_SQL = (() => { + const rendered = new PgDialect().sqlToQuery(boundRunIssueIdSql()); + if (rendered.params.length > 0) throw new Error("bound_run_issue_id_sql_has_parameters"); + return rendered.sql; +})(); + +const APPEND_NEW_NATIVE_SOURCE_EVENTS_WITH_PROGRESS_SQL = ` + with snapshot_run as ( + select next_event_seq from heartbeat_runs where id = $1::uuid + ), allocated as ( + update heartbeat_runs + set next_event_seq = next_event_seq + $6::integer, + updated_at = $7::timestamptz, + native_source_cursors = jsonb_set(coalesce(native_source_cursors, '{}'::jsonb), + array[$4::text], to_jsonb($5::bigint + $6::integer - 1)) + where id = $1::uuid and company_id = $2::uuid and agent_id = $3::uuid + and coalesce((native_source_cursors ->> $4::text)::bigint, 0) = $5::bigint - 1 + and next_event_seq = (select next_event_seq from snapshot_run) + and not exists (select 1 from heartbeat_run_events where run_id = $1::uuid and ( + source_event_id = any(array(select jsonb_array_elements_text($8::jsonb))) + or (source_instance_id = $4::text and source_seq >= $5::bigint))) + returning next_event_seq - $6::integer as first_seq, runtime_mode as wake_runtime_mode, + ${BOUND_RUN_ISSUE_ID_SQL} as wake_issue_id + ), inserted as ( + insert into heartbeat_run_events + (company_id, run_id, agent_id, seq, event_type, stream, level, color, message, payload, + source_instance_id, source_event_id, source_seq, source_payload_sha256, protocol_schema_version) + select $2::uuid, $1::uuid, $3::uuid, allocated.first_seq + event.ordinal - 1, + event.value ->> 'eventType', event.value ->> 'stream', event.value ->> 'level', event.value ->> 'color', + event.value ->> 'message', (event.value ->> 'payload')::jsonb, + $4::text, event.value ->> 'sourceEventId', $5::bigint - 1 + event.ordinal, + event.value ->> 'sourcePayloadSha256', (event.value ->> 'protocolSchemaVersion')::integer + from allocated cross join jsonb_array_elements($9::jsonb) with ordinality as event(value, ordinal) + order by event.ordinal + returning * + ), notified as ( + select pg_notify($11::text, $10::text) from allocated + where $10::text is not null and allocated.wake_runtime_mode = 'native' and allocated.wake_issue_id is not null + ) + select inserted.*, allocated.wake_runtime_mode, allocated.wake_issue_id, (select count(*) from notified) as wake_notified + from inserted cross join allocated`; + async function appendNewNativeSourceEvents( db: Db, inputs: readonly NativeSourceEventInput[], + progressWake = false, ): Promise { const first = inputs[0]!; const source = first.nativeSource; @@ -307,16 +353,24 @@ async function appendNewNativeSourceEvents( sourcePayloadSha256: nativeSha256(input.nativeSource.canonicalPayload), protocolSchemaVersion: input.nativeSource.protocolSchemaVersion, }))); + const parameters = [first.runId, first.companyId, first.agentId, source.sourceInstanceId, String(firstSeq), String(inputs.length), + new Date().toISOString(), eventIds, rows]; let inserted: Record[]; try { - inserted = await db.$client.unsafe[]>(APPEND_NEW_NATIVE_SOURCE_EVENTS_SQL, [first.runId, first.companyId, - first.agentId, source.sourceInstanceId, String(firstSeq), String(inputs.length), new Date().toISOString(), eventIds, rows]); + inserted = progressWake + ? await db.$client.unsafe[]>(APPEND_NEW_NATIVE_SOURCE_EVENTS_WITH_PROGRESS_SQL, [...parameters, + chatNativeProgressCommitNotification(db), DATABASE_WORK_CHANNEL]) + : await db.$client.unsafe[]>(APPEND_NEW_NATIVE_SOURCE_EVENTS_SQL, parameters); } catch (error) { if (isUniqueViolation(error)) return null; throw error; } if (inserted.length === 0) return null; if (inserted.length !== inputs.length) throw new Error("heartbeat_run_event_not_persisted"); + const woken = inserted[0]!; + if (progressWake && woken.wake_runtime_mode === "native" && typeof woken.wake_issue_id === "string") { + publishChatNativeProgressSettled(db, { companyId: first.companyId, issueId: woken.wake_issue_id }); + } const bySourceSeq = new Map(inserted.map(raw => [Number(raw.source_seq), decodeRunEventRow(raw)])); return inputs.map(input => ({ row: bySourceSeq.get(input.nativeSource.sourceSeq)!,