diff --git a/scripts/benchmark-chat-native-progress.mjs b/scripts/benchmark-chat-native-progress.mjs new file mode 100644 index 00000000000..b98d6ac0611 --- /dev/null +++ b/scripts/benchmark-chat-native-progress.mjs @@ -0,0 +1,436 @@ +import assert from "node:assert/strict"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + companies, + agents, + issues, + createDb, + startEmbeddedPostgresTestDatabase +} from "../packages/db/src/index.js"; +const phase = process.argv.find((arg) => arg.startsWith("--phase="))?.slice(8) ?? "baseline"; +assert.ok(phase === "baseline" || phase === "treatment", "--phase must be baseline or treatment"); +const ANCHOR_MS = Date.parse("2026-10-19T00:00:00.000Z"); +const SINCE_MS = ANCHOR_MS - 24 * 60 * 6e4; +const PROMPT_KIB = 144; +function mulberry32(seed2) { + return () => { + seed2 |= 0; + seed2 = seed2 + 1831565813 | 0; + let t = Math.imul(seed2 ^ seed2 >>> 15, 1 | seed2); + t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t; + return ((t ^ t >>> 14) >>> 0) / 4294967296; + }; +} +const rand = mulberry32(1346457664); +const hex = (n) => Array.from({ length: n }, () => "0123456789abcdef"[Math.floor(rand() * 16)]).join(""); +const uuid = () => `${hex(8)}-${hex(4)}-4${hex(3)}-${"89ab"[Math.floor(rand() * 4)]}${hex(3)}-${hex(12)}`; +const prompt = Buffer.from( + Array.from({ length: PROMPT_KIB * 1024 }, () => Math.floor(rand() * 256)) +).toString("base64"); +const iso = (ms) => new Date(ms).toISOString(); +async function seed(db) { + const companyA = uuid(); + const companyB = uuid(); + const agentA = uuid(); + const agentB = uuid(); + const endpointA = uuid(); + const endpointB = uuid(); + await db.insert(companies).values([ + { id: companyA, name: "Synthetic chat polls A", issuePrefix: "BENCH" }, + { id: companyB, name: "Synthetic chat polls B", issuePrefix: "BNCH" } + ]); + await db.insert(agents).values([ + { id: agentA, companyId: companyA, name: "Fixture A", adapterType: "codex_local" }, + { id: agentB, companyId: companyB, name: "Fixture B", adapterType: "codex_local" } + ]); + const conversationIssueIds = []; + const issueIdsA = []; + for (let n = 0; n < 50; n++) { + const issueId = uuid(); + if (n < 45) conversationIssueIds.push(issueId); + issueIdsA.push(issueId); + await db.insert(issues).values({ id: issueId, companyId: companyA, title: `Fixture issue A${n}` }); + } + const conversationIssueIdsB = []; + for (let n = 0; n < 10; n++) { + const issueId = uuid(); + conversationIssueIdsB.push(issueId); + await db.insert(issues).values({ id: issueId, companyId: companyB, title: `Fixture issue B${n}` }); + } + for (const [companyId, endpointId, agentId, ids, label] of [ + [companyA, endpointA, agentA, conversationIssueIds, "A"], + [companyB, endpointB, agentB, conversationIssueIdsB, "B"] + ]) { + const applicationId = uuid(); + const connectionId = uuid(); + await db.$client.unsafe( + `INSERT INTO tool_applications + (id, company_id, name, type, status) + VALUES ($1::uuid, $2::uuid, $3, 'chat', 'active')`, + [applicationId, companyId, `fixture-app-${label}`] + ); + await db.$client.unsafe( + `INSERT INTO tool_connections + (id, company_id, application_id, name, uid, connection_purpose, transport, status, enabled) + VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5, 'channel', 'chat_sdk', 'active', true)`, + [connectionId, companyId, applicationId, `fixture-connection-${label}`, `fixture-conn-${label}`] + ); + await db.$client.unsafe( + `INSERT INTO chat_endpoints + (id, company_id, connection_id, provider, public_id, publication_mode, assigned_agent_id, status) + VALUES ($1::uuid, $2::uuid, $3::uuid, 'slack', $4, 'automatic', $5::uuid, 'active')`, + [endpointId, companyId, connectionId, `fixture-endpoint-${label}`, agentId] + ); + await db.$client.unsafe( + `INSERT INTO chat_conversations + (id, company_id, endpoint_id, issue_id, external_conversation_id, external_label, state) + SELECT gen_random_uuid(), $1::uuid, $2::uuid, issue::uuid, 'conv-' || n, 'Conv ${label} ' || n, + CASE WHEN n % 9 = 0 THEN 'waiting' WHEN n % 11 = 0 THEN 'completed' ELSE 'active' END + FROM (SELECT issue, row_number() OVER () AS n FROM unnest($3::uuid[]) AS issue) sub`, + [companyId, endpointId, ids] + ); + } + const nativeRunIds = []; + const convRowsA = conversationIssueIds.map((issue, n) => ({ issue, n })); + for (const { issue, n } of convRowsA) { + const nativeRunId = uuid(); + nativeRunIds.push(nativeRunId); + const base = ANCHOR_MS - n % 40 * 6e4; + await db.$client.unsafe( + `INSERT INTO heartbeat_runs + (id, company_id, agent_id, scope_kind, issue_id, runtime_mode, status, error_code, created_at, updated_at, context_snapshot, result_json) + VALUES ($1::uuid, $2::uuid, $3::uuid, 'issue', $4::uuid, 'native', 'running', NULL, $5, $5, + jsonb_build_object('issueId', $4::text, 'source', 'chat:slack:fixture', 'prompt', $6::text), NULL)`, + [nativeRunId, companyA, agentA, issue, iso(base), prompt] + ); + await db.$client.unsafe( + `INSERT INTO heartbeat_runs + (id, company_id, agent_id, scope_kind, runtime_mode, status, error_code, created_at, updated_at, context_snapshot, result_json) + VALUES ($1::uuid, $2::uuid, $3::uuid, 'company', 'legacy', + CASE WHEN $4 % 5 = 0 THEN 'failed' WHEN $4 % 7 = 0 THEN 'succeeded' ELSE 'running' END, + CASE WHEN $4 % 5 = 0 THEN 'execution_reconciliation_required' END, $5, $5, + jsonb_build_object('issueId', $6::text, 'source', 'chat:slack:fixture', 'prompt', $7::text), + CASE WHEN $4 % 7 = 0 THEN jsonb_build_object('presentationDecision', jsonb_build_object('kind', 'fixture')) END)`, + [uuid(), companyA, agentA, n, iso(base), issue, prompt] + ); + await db.$client.unsafe( + `INSERT INTO chat_publications + (company_id, endpoint_id, conversation_id, issue_id, idempotency_key, payload, state, created_at) + VALUES ($1::uuid, $2::uuid, + (SELECT id FROM chat_conversations WHERE company_id = $1::uuid AND issue_id = $4::uuid LIMIT 1), + $4::uuid, 'run:' || $3::text || ':working:' || $2::text, + jsonb_build_object('classification', 'external', 'source', 'safe_milestone', 'text', 'working'), + 'streaming', $5)`, + [companyA, endpointA, nativeRunId, issue, iso(base - 12e4)] + ); + await db.$client.unsafe( + `INSERT INTO heartbeat_run_events + (company_id, run_id, agent_id, seq, event_type, message, created_at) + VALUES ($1::uuid, $2::uuid, $3::uuid, 1, 'workspace.ready', 'ready', $4), + ($1::uuid, $2::uuid, $3::uuid, 2, 'tool.execution.started', 'tool', $5)`, + [companyA, nativeRunId, agentA, iso(base + 3e4), iso(base + 45e3)] + ); + } + for (const [n, issue] of conversationIssueIdsB.entries()) { + const runId = uuid(); + const base = ANCHOR_MS - n % 10 * 6e4; + await db.$client.unsafe( + `INSERT INTO heartbeat_runs + (id, company_id, agent_id, scope_kind, issue_id, runtime_mode, status, error_code, created_at, updated_at, context_snapshot) + VALUES ($1::uuid, $2::uuid, $3::uuid, 'issue', $4::uuid, 'native', 'running', NULL, $5, $5, + jsonb_build_object('issueId', $4::text, 'source', 'chat:slack:fixture', 'prompt', $6::text))`, + [runId, companyB, agentB, issue, iso(base), prompt] + ); + } + for (let start = 0; start < 2400; start += 400) { + await db.$client.unsafe( + `INSERT INTO heartbeat_runs + (company_id, agent_id, scope_kind, status, error_code, created_at, updated_at, context_snapshot) + SELECT $1::uuid, $2::uuid, 'company', + (ARRAY['queued','running','succeeded','interrupted','failed','timed_out','cancelled'])[(n % 7) + 1], + NULL, + make_interval(hours => CASE WHEN n % 3 = 0 THEN -(30 + (n % 100)) ELSE -(n % 16) END, secs => -60) + $4::timestamptz, + make_interval(hours => CASE WHEN n % 3 = 0 THEN -(30 + (n % 100)) ELSE -(n % 16) END, secs => -60) + $4::timestamptz, + jsonb_build_object('issueId', gen_random_uuid()::text, 'source', 'chat:slack:fixture', 'prompt', $3::text) + FROM generate_series(${start + 1}, ${start + 400}) n`, + [companyA, agentA, prompt, iso(ANCHOR_MS)] + ); + } + await db.$client.unsafe( + `INSERT INTO heartbeat_runs + (company_id, agent_id, scope_kind, status, error_code, created_at, updated_at, context_snapshot) + SELECT $1::uuid, $2::uuid, 'company', + (ARRAY['queued','running','succeeded','interrupted','failed','timed_out','cancelled'])[(n % 7) + 1], + NULL, + make_interval(hours => CASE WHEN n % 3 = 0 THEN -(30 + (n % 100)) ELSE -(n % 16) END, secs => -60) + $4::timestamptz, + make_interval(hours => CASE WHEN n % 3 = 0 THEN -(30 + (n % 100)) ELSE -(n % 16) END, secs => -60) + $4::timestamptz, + jsonb_build_object('issueId', gen_random_uuid()::text, 'source', 'chat:slack:fixture', 'prompt', $3::text) + FROM generate_series(1, 100) n`, + [companyB, agentB, prompt, iso(ANCHOR_MS)] + ); + await db.$client.unsafe( + "UPDATE heartbeat_runs SET scope_kind = 'company', issue_id = NULL WHERE id = $1::uuid", + [nativeRunIds[0]] + ); + await db.$client.unsafe("ANALYZE heartbeat_runs"); + await db.$client.unsafe("ANALYZE chat_conversations"); + await db.$client.unsafe("ANALYZE heartbeat_run_events"); + await db.$client.unsafe("ANALYZE chat_publications"); + return { companyA, companyB }; +} +function recordingDb(db, sink) { + return new Proxy(db, { + get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields) => { + const builder = target.select(...fields); + const from = builder.from.bind(builder); + builder.from = (...args) => { + const selection = from(...args); + const execute = selection.execute.bind(selection); + selection.execute = async () => { + const captured = selection.toSQL(); + sink.push({ sql: captured.sql, params: captured.params }); + return execute(); + }; + return selection; + }; + return builder; + }; + } + }); +} +async function capture(db) { + const sink = []; + const module = await import("../server/src/services/chat-run-publications.js"); + await module.enqueueChatRunMilestones(recordingDb(db, sink), { + since: new Date(SINCE_MS), + limit: 200 + }); + const milestone = sink.find( + (q) => q.sql.startsWith('select "heartbeat_runs"."id", "heartbeat_runs"."status"') && q.sql.includes('inner join "chat_conversations"') + ); + const native = sink.find((q) => q.sql.startsWith('select "heartbeat_run_events"."id", "heartbeat_run_events"."seq"')); + assert.ok(milestone, "milestone candidate query was not captured"); + assert.ok(native, "native progress candidate query was not captured"); + assert.match(milestone.sql, /chat_conversations/, "milestone query lacks the conversations join"); + assert.match(native.sql, /chat_conversations/, "native query lacks the conversations join"); + return { milestone, native }; +} +async function measureOnce(client, query) { + const start = performance.now(); + const rows = await client.unsafe(query.sql, query.params).values(); + return { ms: performance.now() - start, rows }; +} +async function measure(client, query, rounds = 7) { + const samples = []; + let rows = []; + for (let round = 0; round < rounds; round++) { + const result = await measureOnce(client, query); + rows = result.rows; + if (round >= 2) samples.push(result.ms); + } + const [plan] = await client.unsafe( + `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${query.sql}`, + query.params + ); + samples.sort((a, b) => a - b); + return { samples, rows, plan }; +} +const normalize = (value) => value instanceof Date ? { __date: value.toISOString() } : Array.isArray(value) ? value.map(normalize) : value !== null && typeof value === "object" ? Object.fromEntries(Object.entries(value).map(([k, v]) => [k, normalize(v)])) : value; +function planSummary(plan) { + const nodes = []; + const wrapped = plan?.["QUERY PLAN"]; + const raw = wrapped ?? plan; + const parsed = typeof raw === "string" ? JSON.parse(raw) : raw; + const root = Array.isArray(parsed) ? parsed[0] : parsed; + const planNode = root?.["Plan"] ?? root; + assert.ok(planNode, "explain plan lacks a Plan node"); + const walk = (node) => { + nodes.push({ + node_type: node["Node Type"], + actual_rows: node["Actual Rows"], + loops: node["Actual Loops"] ?? 1, + shared_hit: node["Shared Hit Blocks"], + shared_read: node["Shared Read Blocks"] + }); + for (const child of node["Plans"] ?? []) walk(child); + }; + walk(planNode); + return { root_node: planNode?.["Node Type"], node_count: nodes.length, nodes: nodes.slice(0, 12) }; +} +const median = (values) => values.slice().sort((a, b) => a - b)[Math.floor(values.length / 2)]; +const keepAlive = setInterval(() => { +}, 1e3); +let database; +const databases = []; +async function withDatabase(fn) { + database = await startEmbeddedPostgresTestDatabase(`chat-poll-bench-${phase}-`); + databases.push(database); + const db = createDb(database.connectionString); + try { + return await fn(db); + } finally { + await db.$client.end(); + await database.cleanup(); + } +} +try { + const scratchDir = path.join(fileURLToPath(new URL("../.scratch/", import.meta.url))); + await fs.mkdir(scratchDir, { recursive: true }); + const baselinePath = path.join(scratchDir, "chat-native-progress-baseline.json"); + const reportPath = path.join(scratchDir, "chat-native-progress-report.json"); + const captured = await withDatabase((db) => seed(db).then(() => capture(db))); + if (phase === "baseline") { + const result = await withDatabase(async (db) => { + await seed(db); + const milestone = await measure(db.$client, captured.milestone); + const native = await measure(db.$client, captured.native); + return { milestone, native }; + }); + const payload = { + anchor: new Date(ANCHOR_MS).toISOString(), + since: new Date(SINCE_MS).toISOString(), + promptKiB: PROMPT_KIB, + fixture: { + conversationIssues: 45, + candidateRunsPerIssue: 2, + decoyRuns: 2400, + crossCompanyDecoys: 100, + nativeRunsWithEvents: 45 + }, + milestone: { + sql: captured.milestone.sql, + params: captured.milestone.params, + samples: result.milestone.samples, + medianMs: median(result.milestone.samples), + rows: normalize(result.milestone.rows), + plan: planSummary(result.milestone.plan), + fullPlan: result.milestone.plan + }, + native: { + sql: captured.native.sql, + params: captured.native.params, + samples: result.native.samples, + medianMs: median(result.native.samples), + rows: normalize(result.native.rows), + plan: planSummary(result.native.plan), + fullPlan: result.native.plan + } + }; + await fs.writeFile(baselinePath, JSON.stringify(payload, null, 2)); + console.log( + JSON.stringify( + { + phase, + milestone: { + medianMs: payload.milestone.medianMs, + samples: payload.milestone.samples, + rows: result.milestone.rows.length, + plan: payload.milestone.plan.root_node + }, + native: { + medianMs: payload.native.medianMs, + samples: payload.native.samples, + rows: result.native.rows.length, + plan: payload.native.plan.root_node + } + }, + null, + 2 + ) + ); + } else { + const baseline = JSON.parse(await fs.readFile(baselinePath, "utf8")); + const report = await withDatabase(async (db) => { + await seed(db); + const client = db.$client; + const pairs = { + milestone: { baseline: { sql: baseline.milestone.sql, params: baseline.milestone.params }, treatment: captured.milestone }, + native: { baseline: { sql: baseline.native.sql, params: baseline.native.params }, treatment: captured.native } + }; + const measurements = { + milestone: { baseline: [], treatment: [] }, + native: { baseline: [], treatment: [] } + }; + const rows = { + milestone: { baseline: [], treatment: [] }, + native: { baseline: [], treatment: [] } + }; + for (let round = 0; round < 7; round++) { + for (const name of ["milestone", "native"]) { + const order = round % 2 === 0 ? ["baseline", "treatment"] : ["treatment", "baseline"]; + for (const side of order) { + const query = pairs[name][side]; + const result = await measureOnce(client, query); + rows[name][side] = result.rows; + if (round >= 2) measurements[name][side].push(result.ms); + } + } + } + const plans = { + milestone: { baseline: null, treatment: null }, + native: { baseline: null, treatment: null } + }; + for (const name of ["milestone", "native"]) { + for (const side of ["baseline", "treatment"]) { + const [plan] = await client.unsafe( + `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${pairs[name][side].sql}`, + pairs[name][side].params + ); + plans[name][side] = planSummary(plan); + } + } + return { measurements, rows, plans }; + }); + for (const name of ["milestone", "native"]) { + assert.deepEqual( + normalize(report.rows[name].treatment), + normalize(report.rows[name].baseline), + `${name} poll rows changed between baseline and treatment` + ); + assert.equal( + report.rows[name].baseline.length, + baseline[name].rows.length, + `${name} poll baseline cardinality drifted across fixtures` + ); + if (name === "native") assert.ok(report.rows[name].baseline.length > 0, "native equality must cover actual rows"); + } + const summary = {}; + for (const name of ["milestone", "native"]) { + for (const side of ["baseline", "treatment"]) { + summary[`${name}:${side}`] = { + medianMs: median(report.measurements[name][side]), + samples: report.measurements[name][side], + rows: report.rows[name][side].length + }; + } + } + const output = { + phase, + fixture: { + promptKiB: PROMPT_KIB, + windowHours: 24, + decoyRuns: 2400, + crossCompanyDecoys: 100 + }, + summary, + plans: report.plans, + speedup: { + milestone: summary["milestone:baseline"].medianMs / summary["milestone:treatment"].medianMs, + native: summary["native:baseline"].medianMs / summary["native:treatment"].medianMs + }, + equalRows: true + }; + await fs.writeFile(reportPath, JSON.stringify(output, null, 2)); + console.log(JSON.stringify({ ...output, plans: void 0 }, null, 2)); + } +} finally { + clearInterval(keepAlive); + for (const database2 of databases) { + await database2.cleanup().catch(() => { + }); + } +} diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index d4fcc72270d..539acdfca90 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -62746,6 +62746,37 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }, ); + it.each([true, false])( + "publishes native progress with a typed issue binding present: %s", + async (typedBinding) => { + const context = await safeNativeProgressFixture("telegram", typedBinding ? "107" : "108"); + try { + const run = await context.createRun("issue binding"); + if (typedBinding) { + await db.update(heartbeatRuns).set({ + scopeKind: "issue", + issueId: context.conversation.issueId, + contextSnapshot: sql`${heartbeatRuns.contextSnapshot} - 'issueId'`, + }).where(and( + eq(heartbeatRuns.companyId, context.fixture.companyId), + eq(heartbeatRuns.id, run.runId), + )); + } + await context.addEvent(run, "tool.execution.started", 1); + await expect(enqueueChatRunMilestones(db, { since: new Date(0) })).resolves.toBe(1); + await expect(db.select({ issueId: chatPublications.issueId }).from(chatPublications).where(and( + eq(chatPublications.companyId, context.fixture.companyId), + eq(chatPublications.conversationId, context.conversation.id), + like(chatPublications.idempotencyKey, `run:${run.runId}:working:${context.endpoint.id}:native:%`), + ))).resolves.toEqual([{ issueId: context.conversation.issueId }]); + await context.service.processPendingPublications(100); + expect(context.providerRuntime.edits).toHaveLength(1); + } finally { + await context.service.shutdown(); + } + }, + ); + it("enforces the native progress cadence boundary and one publication per phase", async () => { const context = await safeNativeProgressFixture("telegram", "8"); const run = await context.createRun("cadence boundaries"); diff --git a/server/src/services/chat-run-publications.ts b/server/src/services/chat-run-publications.ts index f8c4575c2b3..cc57a2c5d37 100644 --- a/server/src/services/chat-run-publications.ts +++ b/server/src/services/chat-run-publications.ts @@ -331,7 +331,7 @@ async function enqueueSafeNativeChatProgress( chatConversations, and( eq(chatConversations.companyId, heartbeatRuns.companyId), - sql`${issueIdFromContext} = ${chatConversations.issueId}::text`, + sql`coalesce(${heartbeatRuns.issueId}::text, ${issueIdFromContext}) = ${chatConversations.issueId}::text`, inArray(chatConversations.state, ["active", "waiting"]), ), )