diff --git a/server/scripts/benchmark-native-progress-page.ts b/server/scripts/benchmark-native-progress-page.ts new file mode 100644 index 00000000000..dc4824db5e4 --- /dev/null +++ b/server/scripts/benchmark-native-progress-page.ts @@ -0,0 +1,158 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { PerformanceObserver, performance, type PerformanceEntry } from "node:perf_hooks"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import v8 from "node:v8"; +import vm from "node:vm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { chatPublications, createDb, heartbeatRunEvents, heartbeatRuns, startEmbeddedPostgresTestDatabase } from "../../packages/db/src/index.js"; +import { createChatMilestoneFixtures } from "../src/__tests__/helpers/chat-milestone-fixtures.js"; +import * as current from "../src/services/chat-native-progress.js"; +import { safeNativeChatProgressForEvent } from "../src/services/safe-native-chat-progress.js"; + +const baselineRef = process.argv.find(arg => arg.startsWith("--baseline-ref="))?.slice(15); +assert.ok(baselineRef, "Pass an explicit pre-fix --baseline-ref="); +const root = fileURLToPath(new URL("../../", import.meta.url)); +const git = (...args: string[]) => execFileSync("git", args, { cwd: root, encoding: "utf8" }); +const baselineRevision = git("rev-parse", "--verify", `${baselineRef}^{commit}`).trim(); +const modulePath = "server/src/services/chat-native-progress.ts"; +const ITERATIONS = Number(process.env.ITERATIONS ?? 2000), PASSES = Number(process.env.PASSES ?? 5); +type Progress = Pick; + +const keepAlive = setInterval(() => {}, 1_000); +let temporary: Awaited> | undefined; +let db: ReturnType | undefined; +let baselineDir: string | undefined; +let collections: PerformanceEntry[] = []; +const gcObserver = new PerformanceObserver(list => { collections.push(...list.getEntries()); }); +gcObserver.observe({ entryTypes: ["gc"] }); +function collectionsBetween(start: number, end: number) { + collections.push(...gcObserver.takeRecords()); + const within = collections.filter(entry => entry.startTime >= start && entry.startTime <= end); + collections = collections.filter(entry => entry.startTime > end); + return { count: within.length, ms: within.reduce((sum, entry) => sum + entry.duration, 0) }; +} +const allocated = () => (v8.getHeapStatistics() as { total_allocated_bytes?: number }).total_allocated_bytes; +v8.setFlagsFromString("--expose-gc"); +const collectGarbage = vm.runInNewContext("gc") as () => void; + +try { + const moduleUrl = new URL(`../../${modulePath}`, import.meta.url); + baselineDir = await fs.mkdtemp(path.join(root, "server/scripts/.native-progress-baseline-")); + const baselineFile = path.join(baselineDir, "chat-native-progress.ts"); + await fs.writeFile(baselineFile, git("show", `${baselineRevision}:${modulePath}`).replace(/from "(\.{1,2}\/[^"]+)"/g, + (_match, specifier: string) => `from ${JSON.stringify(fileURLToPath(new URL(specifier, moduleUrl)))}`)); + const baseline = await import(pathToFileURL(baselineFile).href) as Progress; + const versions: Array<{ label: "baseline" | "treatment"; progress: Progress }> = [ + { label: "baseline", progress: baseline }, { label: "treatment", progress: current }]; + + temporary = await startEmbeddedPostgresTestDatabase("native-progress-page-benchmark-"); + db = createDb(temporary.connectionString); + const database = db; + const fixtures = createChatMilestoneFixtures(database); + const chat = await fixtures.create(); + const runId = randomUUID(), baseKey = `run:${runId}:working:${chat.endpointId}`; + await database.insert(heartbeatRuns).values({ id: runId, companyId: chat.companyId, agentId: chat.agentId, nativeIssueId: chat.issueId, + runtimeMode: "native", status: "running", contextSnapshot: { issueId: chat.issueId } }); + await database.insert(heartbeatRunEvents).values({ companyId: chat.companyId, agentId: chat.agentId, runId, seq: 1, + eventType: "item.completed", createdAt: new Date(Date.now() - 60_000) }); + const phase = safeNativeChatProgressForEvent("item.completed", "Maya")!.phase; + await database.insert(chatPublications).values([ + { companyId: chat.companyId, endpointId: chat.endpointId, conversationId: chat.conversationId, issueId: chat.issueId, + idempotencyKey: baseKey, payload: { text: "Maya is working…", progressState: "working" }, state: "published", + createdAt: new Date(Date.now() - 600_000) }, + { companyId: chat.companyId, endpointId: chat.endpointId, conversationId: chat.conversationId, issueId: chat.issueId, + idempotencyKey: `${baseKey}:native:${phase}:1`, payload: { text: "Maya is making progress…", progressState: "working" }, + state: "published", createdAt: new Date(Date.now() - 30_000) }, + ]); + const scope = { companyId: chat.companyId, issueId: chat.issueId }; + const since = new Date(Date.now() - 3_600_000); + const scenarios = { + page: (progress: Progress) => progress.loadNativeChatProgressScopes(database, scope), + wake: (progress: Progress) => progress.reconcileNativeChatProgress(database, { scope, since, limit: 100 }), + }; + type Scenario = keyof typeof scenarios; + const names = Object.keys(scenarios) as Scenario[]; + + let sent: Array<{ text: string; parameters: unknown[] }> | null = null; + (database.$client.options as { debug?: unknown }).debug = (_connection: number, text: string, parameters: unknown[]) => { + sent?.push({ text, parameters }); + }; + const comparable = (statements: NonNullable) => statements.map(statement => ({ text: statement.text, + parameters: statement.parameters.map(value => typeof value === "string" && /^\d{4}-\d{2}-\d{2}T[\d:.]+Z$/.test(value) ? "