import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import { promises as fs } from "node:fs"; import path from "node:path"; import { PerformanceObserver, performance, type PerformanceEntry } from "node:perf_hooks"; import { fileURLToPath, pathToFileURL } from "node:url"; import v8 from "node:v8"; import vm from "node:vm"; import { PgDialect } from "drizzle-orm/pg-core"; import { chatPublications, createDb, heartbeatRunEvents, heartbeatRuns, startEmbeddedPostgresTestDatabase } from "../../packages/db/src/index.js"; import { createChatMilestoneFixtures } from "../src/__tests__/helpers/chat-milestone-fixtures.js"; import * as current from "../src/services/chat-native-progress.js"; import { safeNativeChatProgressForEvent } from "../src/services/safe-native-chat-progress.js"; const baselineRef = process.argv.find(arg => arg.startsWith("--baseline-ref="))?.slice(15); assert.ok(baselineRef, "Pass an explicit pre-fix --baseline-ref="); const root = fileURLToPath(new URL("../../", import.meta.url)); const git = (...args: string[]) => execFileSync("git", args, { cwd: root, encoding: "utf8" }); const baselineRevision = git("rev-parse", "--verify", `${baselineRef}^{commit}`).trim(); const modulePath = "server/src/services/chat-native-progress.ts"; const ITERATIONS = Number(process.env.ITERATIONS ?? 2000), PASSES = Number(process.env.PASSES ?? 5); type Progress = Pick; const keepAlive = setInterval(() => {}, 1_000); let temporary: Awaited> | undefined; let db: ReturnType | undefined; let baselineDir: string | undefined; let collections: PerformanceEntry[] = []; const gcObserver = new PerformanceObserver(list => { collections.push(...list.getEntries()); }); gcObserver.observe({ entryTypes: ["gc"] }); function collectionsBetween(start: number, end: number) { collections.push(...gcObserver.takeRecords()); const within = collections.filter(entry => entry.startTime >= start && entry.startTime <= end); collections = collections.filter(entry => entry.startTime > end); return { count: within.length, ms: within.reduce((sum, entry) => sum + entry.duration, 0) }; } const allocated = () => (v8.getHeapStatistics() as { total_allocated_bytes?: number }).total_allocated_bytes; v8.setFlagsFromString("--expose-gc"); const collectGarbage = vm.runInNewContext("gc") as () => void; try { const moduleUrl = new URL(`../../${modulePath}`, import.meta.url); baselineDir = await fs.mkdtemp(path.join(root, "server/scripts/.native-progress-baseline-")); const baselineFile = path.join(baselineDir, "chat-native-progress.ts"); await fs.writeFile(baselineFile, git("show", `${baselineRevision}:${modulePath}`).replace(/from "(\.{1,2}\/[^"]+)"/g, (_match, specifier: string) => `from ${JSON.stringify(fileURLToPath(new URL(specifier, moduleUrl)))}`)); const baseline = await import(pathToFileURL(baselineFile).href) as Progress; const versions: Array<{ label: "baseline" | "treatment"; progress: Progress }> = [ { label: "baseline", progress: baseline }, { label: "treatment", progress: current }]; temporary = await startEmbeddedPostgresTestDatabase("native-progress-page-benchmark-"); db = createDb(temporary.connectionString); const database = db; const fixtures = createChatMilestoneFixtures(database); const chat = await fixtures.create(); const runId = randomUUID(), baseKey = `run:${runId}:working:${chat.endpointId}`; await database.insert(heartbeatRuns).values({ id: runId, companyId: chat.companyId, agentId: chat.agentId, nativeIssueId: chat.issueId, runtimeMode: "native", status: "running", contextSnapshot: { issueId: chat.issueId } }); await database.insert(heartbeatRunEvents).values({ companyId: chat.companyId, agentId: chat.agentId, runId, seq: 1, eventType: "item.completed", createdAt: new Date(Date.now() - 60_000) }); const phase = safeNativeChatProgressForEvent("item.completed", "Maya")!.phase; await database.insert(chatPublications).values([ { companyId: chat.companyId, endpointId: chat.endpointId, conversationId: chat.conversationId, issueId: chat.issueId, idempotencyKey: baseKey, payload: { text: "Maya is working…", progressState: "working" }, state: "published", createdAt: new Date(Date.now() - 600_000) }, { companyId: chat.companyId, endpointId: chat.endpointId, conversationId: chat.conversationId, issueId: chat.issueId, idempotencyKey: `${baseKey}:native:${phase}:1`, payload: { text: "Maya is making progress…", progressState: "working" }, state: "published", createdAt: new Date(Date.now() - 30_000) }, ]); const scope = { companyId: chat.companyId, issueId: chat.issueId }; const since = new Date(Date.now() - 3_600_000); const scenarios = { page: (progress: Progress) => progress.loadNativeChatProgressScopes(database, scope), wake: (progress: Progress) => progress.reconcileNativeChatProgress(database, { scope, since, limit: 100 }), }; type Scenario = keyof typeof scenarios; const names = Object.keys(scenarios) as Scenario[]; let sent: Array<{ text: string; parameters: unknown[] }> | null = null; (database.$client.options as { debug?: unknown }).debug = (_connection: number, text: string, parameters: unknown[]) => { sent?.push({ text, parameters }); }; const comparable = (statements: NonNullable) => statements.map(statement => ({ text: statement.text, parameters: statement.parameters.map(value => typeof value === "string" && /^\d{4}-\d{2}-\d{2}T[\d:.]+Z$/.test(value) ? "