PaperClipAI/scripts/benchmark-chat-native-progress.mjs

import assert from "node:assert/strict";
import { promises as fs } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import {
  companies,
  agents,
  issues,
  createDb,
  startEmbeddedPostgresTestDatabase
} from "../packages/db/src/index.js";
const phase = process.argv.find((arg) => arg.startsWith("--phase="))?.slice(8) ?? "baseline";
assert.ok(phase === "baseline" || phase === "treatment", "--phase must be baseline or treatment");
const ANCHOR_MS = Date.parse("2026-10-19T00:00:00.000Z");
const SINCE_MS = ANCHOR_MS - 24 * 60 * 6e4;
const PROMPT_KIB = 144;
function mulberry32(seed2) {
  return () => {
    seed2 |= 0;
    seed2 = seed2 + 1831565813 | 0;
    let t = Math.imul(seed2 ^ seed2 >>> 15, 1 | seed2);
    t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
    return ((t ^ t >>> 14) >>> 0) / 4294967296;
  };
}
const rand = mulberry32(1346457664);
const hex = (n) => Array.from({ length: n }, () => "0123456789abcdef"[Math.floor(rand() * 16)]).join("");
const uuid = () => `${hex(8)}-${hex(4)}-4${hex(3)}-${"89ab"[Math.floor(rand() * 4)]}${hex(3)}-${hex(12)}`;
const prompt = Buffer.from(
  Array.from({ length: PROMPT_KIB * 1024 }, () => Math.floor(rand() * 256))
).toString("base64");
const iso = (ms) => new Date(ms).toISOString();
async function seed(db) {
  const companyA = uuid();
  const companyB = uuid();
  const agentA = uuid();
  const agentB = uuid();
  const endpointA = uuid();
  const endpointB = uuid();
  await db.insert(companies).values([
    { id: companyA, name: "Synthetic chat polls A", issuePrefix: "BENCH" },
    { id: companyB, name: "Synthetic chat polls B", issuePrefix: "BNCH" }
  ]);
  await db.insert(agents).values([
    { id: agentA, companyId: companyA, name: "Fixture A", adapterType: "codex_local" },
    { id: agentB, companyId: companyB, name: "Fixture B", adapterType: "codex_local" }
  ]);
  const conversationIssueIds = [];
  const issueIdsA = [];
  for (let n = 0; n < 50; n++) {
    const issueId = uuid();
    if (n < 45) conversationIssueIds.push(issueId);
    issueIdsA.push(issueId);
    await db.insert(issues).values({ id: issueId, companyId: companyA, title: `Fixture issue A${n}` });
  }
  const conversationIssueIdsB = [];
  for (let n = 0; n < 10; n++) {
    const issueId = uuid();
    conversationIssueIdsB.push(issueId);
    await db.insert(issues).values({ id: issueId, companyId: companyB, title: `Fixture issue B${n}` });
  }
  for (const [companyId, endpointId, agentId, ids, label] of [
    [companyA, endpointA, agentA, conversationIssueIds, "A"],
    [companyB, endpointB, agentB, conversationIssueIdsB, "B"]
  ]) {
    const applicationId = uuid();
    const connectionId = uuid();
    await db.$client.unsafe(
      `INSERT INTO tool_applications
        (id, company_id, name, type, status)
       VALUES ($1::uuid, $2::uuid, $3, 'chat', 'active')`,
      [applicationId, companyId, `fixture-app-${label}`]
    );
    await db.$client.unsafe(
      `INSERT INTO tool_connections
        (id, company_id, application_id, name, uid, connection_purpose, transport, status, enabled)
       VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5, 'channel', 'chat_sdk', 'active', true)`,
      [connectionId, companyId, applicationId, `fixture-connection-${label}`, `fixture-conn-${label}`]
    );
    await db.$client.unsafe(
      `INSERT INTO chat_endpoints
        (id, company_id, connection_id, provider, public_id, publication_mode, assigned_agent_id, status)
       VALUES ($1::uuid, $2::uuid, $3::uuid, 'slack', $4, 'automatic', $5::uuid, 'active')`,
      [endpointId, companyId, connectionId, `fixture-endpoint-${label}`, agentId]
    );
    await db.$client.unsafe(
      `INSERT INTO chat_conversations
        (id, company_id, endpoint_id, issue_id, external_conversation_id, external_label, state)
       SELECT gen_random_uuid(), $1::uuid, $2::uuid, issue::uuid, 'conv-' || n, 'Conv ${label} ' || n,
         CASE WHEN n % 9 = 0 THEN 'waiting' WHEN n % 11 = 0 THEN 'completed' ELSE 'active' END
       FROM (SELECT issue, row_number() OVER () AS n FROM unnest($3::uuid[]) AS issue) sub`,
      [companyId, endpointId, ids]
    );
  }
  const nativeRunIds = [];
  const convRowsA = conversationIssueIds.map((issue, n) => ({ issue, n }));
  for (const { issue, n } of convRowsA) {
    const nativeRunId = uuid();
    nativeRunIds.push(nativeRunId);
    const base = ANCHOR_MS - n % 40 * 6e4;
    await db.$client.unsafe(
      `INSERT INTO heartbeat_runs
        (id, company_id, agent_id, scope_kind, issue_id, runtime_mode, status, error_code, created_at, updated_at, context_snapshot, result_json)
       VALUES ($1::uuid, $2::uuid, $3::uuid, 'issue', $4::uuid, 'native', 'running', NULL, $5, $5,
         jsonb_build_object('issueId', $4::text, 'source', 'chat:slack:fixture', 'prompt', $6::text), NULL)`,
      [nativeRunId, companyA, agentA, issue, iso(base), prompt]
    );
    await db.$client.unsafe(
      `INSERT INTO heartbeat_runs
        (id, company_id, agent_id, scope_kind, runtime_mode, status, error_code, created_at, updated_at, context_snapshot, result_json)
       VALUES ($1::uuid, $2::uuid, $3::uuid, 'company', 'legacy',
         CASE WHEN $4 % 5 = 0 THEN 'failed' WHEN $4 % 7 = 0 THEN 'succeeded' ELSE 'running' END,
         CASE WHEN $4 % 5 = 0 THEN 'execution_reconciliation_required' END, $5, $5,
         jsonb_build_object('issueId', $6::text, 'source', 'chat:slack:fixture', 'prompt', $7::text),
         CASE WHEN $4 % 7 = 0 THEN jsonb_build_object('presentationDecision', jsonb_build_object('kind', 'fixture')) END)`,
      [uuid(), companyA, agentA, n, iso(base), issue, prompt]
    );
    await db.$client.unsafe(
      `INSERT INTO chat_publications
        (company_id, endpoint_id, conversation_id, issue_id, idempotency_key, payload, state, created_at)
       VALUES ($1::uuid, $2::uuid,
         (SELECT id FROM chat_conversations WHERE company_id = $1::uuid AND issue_id = $4::uuid LIMIT 1),
         $4::uuid, 'run:' || $3::text || ':working:' || $2::text,
         jsonb_build_object('classification', 'external', 'source', 'safe_milestone', 'text', 'working'),
         'streaming', $5)`,
      [companyA, endpointA, nativeRunId, issue, iso(base - 12e4)]
    );
    await db.$client.unsafe(
      `INSERT INTO heartbeat_run_events
        (company_id, run_id, agent_id, seq, event_type, message, created_at)
       VALUES ($1::uuid, $2::uuid, $3::uuid, 1, 'workspace.ready', 'ready', $4),
              ($1::uuid, $2::uuid, $3::uuid, 2, 'tool.execution.started', 'tool', $5)`,
      [companyA, nativeRunId, agentA, iso(base + 3e4), iso(base + 45e3)]
    );
  }
  for (const [n, issue] of conversationIssueIdsB.entries()) {
    const runId = uuid();
    const base = ANCHOR_MS - n % 10 * 6e4;
    await db.$client.unsafe(
      `INSERT INTO heartbeat_runs
        (id, company_id, agent_id, scope_kind, issue_id, runtime_mode, status, error_code, created_at, updated_at, context_snapshot)
       VALUES ($1::uuid, $2::uuid, $3::uuid, 'issue', $4::uuid, 'native', 'running', NULL, $5, $5,
         jsonb_build_object('issueId', $4::text, 'source', 'chat:slack:fixture', 'prompt', $6::text))`,
      [runId, companyB, agentB, issue, iso(base), prompt]
    );
  }
  for (let start = 0; start < 2400; start += 400) {
    await db.$client.unsafe(
      `INSERT INTO heartbeat_runs
        (company_id, agent_id, scope_kind, status, error_code, created_at, updated_at, context_snapshot)
       SELECT $1::uuid, $2::uuid, 'company',
         (ARRAY['queued','running','succeeded','interrupted','failed','timed_out','cancelled'])[(n % 7) + 1],
         NULL,
         make_interval(hours => CASE WHEN n % 3 = 0 THEN -(30 + (n % 100)) ELSE -(n % 16) END, secs => -60) + $4::timestamptz,
         make_interval(hours => CASE WHEN n % 3 = 0 THEN -(30 + (n % 100)) ELSE -(n % 16) END, secs => -60) + $4::timestamptz,
         jsonb_build_object('issueId', gen_random_uuid()::text, 'source', 'chat:slack:fixture', 'prompt', $3::text)
       FROM generate_series(${start + 1}, ${start + 400}) n`,
      [companyA, agentA, prompt, iso(ANCHOR_MS)]
    );
  }
  await db.$client.unsafe(
    `INSERT INTO heartbeat_runs
      (company_id, agent_id, scope_kind, status, error_code, created_at, updated_at, context_snapshot)
     SELECT $1::uuid, $2::uuid, 'company',
       (ARRAY['queued','running','succeeded','interrupted','failed','timed_out','cancelled'])[(n % 7) + 1],
       NULL,
       make_interval(hours => CASE WHEN n % 3 = 0 THEN -(30 + (n % 100)) ELSE -(n % 16) END, secs => -60) + $4::timestamptz,
       make_interval(hours => CASE WHEN n % 3 = 0 THEN -(30 + (n % 100)) ELSE -(n % 16) END, secs => -60) + $4::timestamptz,
       jsonb_build_object('issueId', gen_random_uuid()::text, 'source', 'chat:slack:fixture', 'prompt', $3::text)
     FROM generate_series(1, 100) n`,
    [companyB, agentB, prompt, iso(ANCHOR_MS)]
  );
  await db.$client.unsafe(
    "UPDATE heartbeat_runs SET scope_kind = 'company', issue_id = NULL WHERE id = $1::uuid",
    [nativeRunIds[0]]
  );
  await db.$client.unsafe("ANALYZE heartbeat_runs");
  await db.$client.unsafe("ANALYZE chat_conversations");
  await db.$client.unsafe("ANALYZE heartbeat_run_events");
  await db.$client.unsafe("ANALYZE chat_publications");
  return { companyA, companyB };
}
function recordingDb(db, sink) {
  return new Proxy(db, {
    get(target, key) {
      if (key !== "select") return Reflect.get(target, key);
      return (...fields) => {
        const builder = target.select(...fields);
        const from = builder.from.bind(builder);
        builder.from = (...args) => {
          const selection = from(...args);
          const execute = selection.execute.bind(selection);
          selection.execute = async () => {
            const captured = selection.toSQL();
            sink.push({ sql: captured.sql, params: captured.params });
            return execute();
          };
          return selection;
        };
        return builder;
      };
    }
  });
}
async function capture(db) {
  const sink = [];
  const module = await import("../server/src/services/chat-run-publications.js");
  await module.enqueueChatRunMilestones(recordingDb(db, sink), {
    since: new Date(SINCE_MS),
    limit: 200
  });
  const milestone = sink.find(
    (q) => q.sql.startsWith('select "heartbeat_runs"."id", "heartbeat_runs"."status"') && q.sql.includes('inner join "chat_conversations"')
  );
  const native = sink.find((q) => q.sql.startsWith('select "heartbeat_run_events"."id", "heartbeat_run_events"."seq"'));
  assert.ok(milestone, "milestone candidate query was not captured");
  assert.ok(native, "native progress candidate query was not captured");
  assert.match(milestone.sql, /chat_conversations/, "milestone query lacks the conversations join");
  assert.match(native.sql, /chat_conversations/, "native query lacks the conversations join");
  return { milestone, native };
}
async function measureOnce(client, query) {
  const start = performance.now();
  const rows = await client.unsafe(query.sql, query.params).values();
  return { ms: performance.now() - start, rows };
}
async function measure(client, query, rounds = 7) {
  const samples = [];
  let rows = [];
  for (let round = 0; round < rounds; round++) {
    const result = await measureOnce(client, query);
    rows = result.rows;
    if (round >= 2) samples.push(result.ms);
  }
  const [plan] = await client.unsafe(
    `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${query.sql}`,
    query.params
  );
  samples.sort((a, b) => a - b);
  return { samples, rows, plan };
}
const normalize = (value) => value instanceof Date ? { __date: value.toISOString() } : Array.isArray(value) ? value.map(normalize) : value !== null && typeof value === "object" ? Object.fromEntries(Object.entries(value).map(([k, v]) => [k, normalize(v)])) : value;
function planSummary(plan) {
  const nodes = [];
  const wrapped = plan?.["QUERY PLAN"];
  const raw = wrapped ?? plan;
  const parsed = typeof raw === "string" ? JSON.parse(raw) : raw;
  const root = Array.isArray(parsed) ? parsed[0] : parsed;
  const planNode = root?.["Plan"] ?? root;
  assert.ok(planNode, "explain plan lacks a Plan node");
  const walk = (node) => {
    nodes.push({
      node_type: node["Node Type"],
      actual_rows: node["Actual Rows"],
      loops: node["Actual Loops"] ?? 1,
      shared_hit: node["Shared Hit Blocks"],
      shared_read: node["Shared Read Blocks"]
    });
    for (const child of node["Plans"] ?? []) walk(child);
  };
  walk(planNode);
  return { root_node: planNode?.["Node Type"], node_count: nodes.length, nodes: nodes.slice(0, 12) };
}
const median = (values) => values.slice().sort((a, b) => a - b)[Math.floor(values.length / 2)];
const keepAlive = setInterval(() => {
}, 1e3);
let database;
const databases = [];
async function withDatabase(fn) {
  database = await startEmbeddedPostgresTestDatabase(`chat-poll-bench-${phase}-`);
  databases.push(database);
  const db = createDb(database.connectionString);
  try {
    return await fn(db);
  } finally {
    await db.$client.end();
    await database.cleanup();
  }
}
try {
  const scratchDir = path.join(fileURLToPath(new URL("../.scratch/", import.meta.url)));
  await fs.mkdir(scratchDir, { recursive: true });
  const baselinePath = path.join(scratchDir, "chat-native-progress-baseline.json");
  const reportPath = path.join(scratchDir, "chat-native-progress-report.json");
  const captured = await withDatabase((db) => seed(db).then(() => capture(db)));
  if (phase === "baseline") {
    const result = await withDatabase(async (db) => {
      await seed(db);
      const milestone = await measure(db.$client, captured.milestone);
      const native = await measure(db.$client, captured.native);
      return { milestone, native };
    });
    const payload = {
      anchor: new Date(ANCHOR_MS).toISOString(),
      since: new Date(SINCE_MS).toISOString(),
      promptKiB: PROMPT_KIB,
      fixture: {
        conversationIssues: 45,
        candidateRunsPerIssue: 2,
        decoyRuns: 2400,
        crossCompanyDecoys: 100,
        nativeRunsWithEvents: 45
      },
      milestone: {
        sql: captured.milestone.sql,
        params: captured.milestone.params,
        samples: result.milestone.samples,
        medianMs: median(result.milestone.samples),
        rows: normalize(result.milestone.rows),
        plan: planSummary(result.milestone.plan),
        fullPlan: result.milestone.plan
      },
      native: {
        sql: captured.native.sql,
        params: captured.native.params,
        samples: result.native.samples,
        medianMs: median(result.native.samples),
        rows: normalize(result.native.rows),
        plan: planSummary(result.native.plan),
        fullPlan: result.native.plan
      }
    };
    await fs.writeFile(baselinePath, JSON.stringify(payload, null, 2));
    console.log(
      JSON.stringify(
        {
          phase,
          milestone: {
            medianMs: payload.milestone.medianMs,
            samples: payload.milestone.samples,
            rows: result.milestone.rows.length,
            plan: payload.milestone.plan.root_node
          },
          native: {
            medianMs: payload.native.medianMs,
            samples: payload.native.samples,
            rows: result.native.rows.length,
            plan: payload.native.plan.root_node
          }
        },
        null,
        2
      )
    );
  } else {
    const baseline = JSON.parse(await fs.readFile(baselinePath, "utf8"));
    const report = await withDatabase(async (db) => {
      await seed(db);
      const client = db.$client;
      const pairs = {
        milestone: { baseline: { sql: baseline.milestone.sql, params: baseline.milestone.params }, treatment: captured.milestone },
        native: { baseline: { sql: baseline.native.sql, params: baseline.native.params }, treatment: captured.native }
      };
      const measurements = {
        milestone: { baseline: [], treatment: [] },
        native: { baseline: [], treatment: [] }
      };
      const rows = {
        milestone: { baseline: [], treatment: [] },
        native: { baseline: [], treatment: [] }
      };
      for (let round = 0; round < 7; round++) {
        for (const name of ["milestone", "native"]) {
          const order = round % 2 === 0 ? ["baseline", "treatment"] : ["treatment", "baseline"];
          for (const side of order) {
            const query = pairs[name][side];
            const result = await measureOnce(client, query);
            rows[name][side] = result.rows;
            if (round >= 2) measurements[name][side].push(result.ms);
          }
        }
      }
      const plans = {
        milestone: { baseline: null, treatment: null },
        native: { baseline: null, treatment: null }
      };
      for (const name of ["milestone", "native"]) {
        for (const side of ["baseline", "treatment"]) {
          const [plan] = await client.unsafe(
            `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${pairs[name][side].sql}`,
            pairs[name][side].params
          );
          plans[name][side] = planSummary(plan);
        }
      }
      return { measurements, rows, plans };
    });
    for (const name of ["milestone", "native"]) {
      assert.deepEqual(
        normalize(report.rows[name].treatment),
        normalize(report.rows[name].baseline),
        `${name} poll rows changed between baseline and treatment`
      );
      assert.equal(
        report.rows[name].baseline.length,
        baseline[name].rows.length,
        `${name} poll baseline cardinality drifted across fixtures`
      );
      if (name === "native") assert.ok(report.rows[name].baseline.length > 0, "native equality must cover actual rows");
    }
    const summary = {};
    for (const name of ["milestone", "native"]) {
      for (const side of ["baseline", "treatment"]) {
        summary[`${name}:${side}`] = {
          medianMs: median(report.measurements[name][side]),
          samples: report.measurements[name][side],
          rows: report.rows[name][side].length
        };
      }
    }
    const output = {
      phase,
      fixture: {
        promptKiB: PROMPT_KIB,
        windowHours: 24,
        decoyRuns: 2400,
        crossCompanyDecoys: 100
      },
      summary,
      plans: report.plans,
      speedup: {
        milestone: summary["milestone:baseline"].medianMs / summary["milestone:treatment"].medianMs,
        native: summary["native:baseline"].medianMs / summary["native:treatment"].medianMs
      },
      equalRows: true
    };
    await fs.writeFile(reportPath, JSON.stringify(output, null, 2));
    console.log(JSON.stringify({ ...output, plans: void 0 }, null, 2));
  }
} finally {
  clearInterval(keepAlive);
  for (const database2 of databases) {
    await database2.cleanup().catch(() => {
    });
  }
}