diff --git a/scripts/benchmark-github-response-run-state.ts b/scripts/benchmark-github-response-run-state.ts new file mode 100644 index 00000000000..9c463f15d75 --- /dev/null +++ b/scripts/benchmark-github-response-run-state.ts @@ -0,0 +1,172 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { createDb, startEmbeddedPostgresTestDatabase, type Db } from "../packages/db/src/index.js"; +import { readGitHubResponseRun } from "../server/src/services/chat-channels/github/response-run-state.js"; + +const baselineRef = process.argv.find(arg => arg.startsWith("--baseline-ref="))?.slice(15); +assert.ok(baselineRef, "Pass an explicit pre-fix --baseline-ref="); +const number = (name: string, fallback: number) => Number(process.env[name] ?? fallback); +const fixture = { + agents: number("BENCH_AGENTS", 4), + runsPerAgent: number("BENCH_RUNS_PER_AGENT", 7_750), + issues: number("BENCH_ISSUES", 2_000), + requests: number("BENCH_REQUESTS", 300), + contextPadBytes: number("BENCH_CONTEXT_PAD_BYTES", 61_440), +}; +const iterations = number("BENCH_ITERATIONS", 7); +const sampleInputs = number("BENCH_SAMPLE_INPUTS", 20); +const baselineRevision = execFileSync("git", ["rev-parse", "--verify", `${baselineRef}^{commit}`], { encoding: "utf8" }).trim(); +const treatmentRevision = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(); +const modulePath = "server/src/services/chat-channels/github/response-run-state.ts"; +const root = fileURLToPath(new URL("../", import.meta.url)); +const keepAlive = setInterval(() => {}, 1_000); +let temporary: Awaited> | undefined; +let db: Db | undefined; +let baselineDir: string | undefined; +type Input = Parameters[1]; + +async function seed(db: Db) { + const sql = db.$client; + const [{ company }] = await sql.unsafe(`insert into companies (name, issue_prefix) values ('Bench', 'BN') returning id as company`); + const agentRows = await sql.unsafe(`insert into agents (company_id, name) select $1, 'Agent ' || n from generate_series(1, ${fixture.agents}) n + returning id`, [company]); + const agentIds = agentRows.map(row => row.id as string); + const [{ application }] = await sql.unsafe(`insert into tool_applications (company_id, application_key, name, type) + values ($1, 'github', 'GitHub', 'chat') returning id as application`, [company]); + const [{ connection }] = await sql.unsafe(`insert into tool_connections (company_id, application_id, name, uid, transport, connection_purpose) + values ($1, $2, 'GitHub', 'github-bench', 'chat_sdk', 'channel') returning id as connection`, [company, application]); + const [{ endpoint }] = await sql.unsafe(`insert into chat_endpoints (company_id, connection_id, provider, public_id, assigned_agent_id, status) + values ($1, $2, 'github', 'bench', $3, 'active') returning id as endpoint`, [company, connection, agentIds[0]]); + await sql.unsafe(`create unlogged table bench_issue (n int primary key, id uuid)`); + await sql.unsafe(`with created as (insert into issues (company_id, title, status) + select $1, 'Issue ' || n, 'in_progress' from generate_series(1, ${fixture.issues}) n returning id, title) + insert into bench_issue select (regexp_replace(title, '\\D', '', 'g'))::int, id from created`, [company]); + await sql.unsafe(`insert into chat_conversations (company_id, endpoint_id, issue_id, external_conversation_id, external_label, external_thread_id) + select $1, $2, i.id, 'github:paperclipai/paperclip:' || i.n, 'PR #' || i.n, 'github:paperclipai/paperclip:' || i.n + from bench_issue i where i.n <= ${fixture.requests}`, [company, endpoint]); + await sql.unsafe(`insert into chat_deliveries (company_id, endpoint_id, provider_event_id, deduplication_key, event_kind, + normalized_event, conversation_id, state) + select $1, $2, 'event-' || i.n, 'dedupe-' || i.n, 'message', '{}', c.id, 'processed' + from bench_issue i join chat_conversations c on c.issue_id = i.id`, [company, endpoint]); + await sql.unsafe(`insert into issue_comments (company_id, issue_id, body) + select $1, i.id, 'Request ' || i.n from bench_issue i where i.n <= ${fixture.requests}`, [company]); + await sql.unsafe(`insert into chat_message_links (company_id, endpoint_id, conversation_id, delivery_id, comment_id, provider_message_id, direction) + select $1, $2, c.id, d.id, ic.id, 'IC_in_' || i.n, 'inbound' + from bench_issue i join chat_conversations c on c.issue_id = i.id join chat_deliveries d on d.conversation_id = c.id + join issue_comments ic on ic.issue_id = i.id`, [company, endpoint]); + await sql.unsafe(`create unlogged table bench_agent (n int primary key, id uuid)`); + for (const [index, id] of agentIds.entries()) await sql.unsafe(`insert into bench_agent values ($1, $2)`, [index, id]); + for (let batch = 0; batch < fixture.agents * fixture.runsPerAgent; batch += 1_000) { + await sql.unsafe(`insert into heartbeat_runs (company_id, agent_id, status, scope_kind, issue_id, native_issue_id, context_snapshot, created_at) + select $1, a.id, case when r % 9 = 0 then 'running' else 'succeeded' end, + case when abs(hashint4(r + 31)::bigint) % 2 = 0 then 'issue' else 'company' end, + case when abs(hashint4(r + 31)::bigint) % 2 = 0 then i.id end, case when abs(hashint4(r + 31)::bigint) % 2 = 0 then i.id end, + jsonb_build_object('issueId', i.id::text, + 'wakeCommentId', (select ic.id::text from issue_comments ic where ic.issue_id = i.id and r % 3 = 0), + 'wakeCommentIds', case when r % 5 = 0 then (select jsonb_agg(ic.id::text) from issue_comments ic where ic.issue_id = i.id) end, + 'pad', (select string_agg(md5(random()::text || g::text || r::text), '') from generate_series(1, ${Math.ceil(fixture.contextPadBytes / 32)}) g)), + timestamptz '2026-01-01' + r * interval '1 minute' + from generate_series($2::int, $3::int) r + join bench_agent a on a.n = abs(hashint4(r + 17)::bigint) % ${fixture.agents} + join bench_issue i on i.n = 1 + abs(hashint4(r)::bigint) % ${fixture.issues}`, + [company, batch, Math.min(batch + 999, fixture.agents * fixture.runsPerAgent - 1)]); + } + const requests = await sql.unsafe(`select i.id as issue, d.id as delivery, ic.id as comment from bench_issue i + join chat_conversations c on c.issue_id = i.id join chat_deliveries d on d.conversation_id = c.id + join issue_comments ic on ic.issue_id = i.id order by i.n`); + await sql.unsafe("drop table bench_issue; drop table bench_agent"); + const inputs: Input[] = requests.flatMap(row => agentIds.slice(0, 2).map(agentId => ({ companyId: company as string, + endpointId: endpoint as string, agentId, issueId: row.issue as string, deliveryId: row.delivery as string, + sourceMessageId: row.comment as string }))); + return { company, inputs }; +} + +try { + const oldSource = execFileSync("git", ["show", `${baselineRevision}:${modulePath}`], { encoding: "utf8" }); + const moduleUrl = new URL(`../${modulePath}`, import.meta.url); + baselineDir = await fs.mkdtemp(path.join(root, "server/scripts/.response-run-state-baseline-")); + const baselineFile = path.join(baselineDir, "response-run-state.ts"); + await fs.writeFile(baselineFile, oldSource.replace(/from "(\.{1,2}\/[^\"]+)"/g, + (_match, specifier: string) => `from ${JSON.stringify(fileURLToPath(new URL(specifier, moduleUrl)))}`)); + const baseline = await import(pathToFileURL(baselineFile).href) as typeof import("../server/src/services/chat-channels/github/response-run-state.js"); + temporary = await startEmbeddedPostgresTestDatabase("github-response-run-state-benchmark-"); + db = createDb(temporary.connectionString); + const seedStart = performance.now(); + const { company, inputs } = await seed(db); + await db.$client.unsafe("ANALYZE"); + const seedMs = performance.now() - seedStart; + + const before = [], after = []; + for (const input of inputs) { + before.push(await baseline.readGitHubResponseRun(db, input)); + after.push(await readGitHubResponseRun(db, input)); + } + assert.deepEqual(after, before, "Every input must return the same latest run and ownership"); + const outcomes = { owns: before.filter(result => result.ownsLatestRequest).length, + latestNotOwned: before.filter(result => result.latestRun && !result.ownsLatestRequest).length, + noRun: before.filter(result => !result.latestRun).length }; + + const withRuns = inputs.filter((_, index) => before[index]!.latestRun); + const sample = withRuns.filter((_, index) => index % Math.max(1, Math.floor(withRuns.length / sampleInputs)) === 0).slice(0, sampleInputs); + const sampledAgents = new Set(sample.map(input => input.agentId)).size; + const versions = { baseline: baseline.readGitHubResponseRun, treatment: readGitHubResponseRun }; + async function timed(version: keyof typeof versions) { + const start = performance.now(); + for (const input of sample) await versions[version](db!, input); + return (performance.now() - start) / sample.length; + } + const samples = { baseline: [] as number[], treatment: [] as number[] }; + for (let n = 0; n < iterations + 2; n++) { + const order = n % 2 ? ["treatment", "baseline"] as const : ["baseline", "treatment"] as const; + for (const version of order) { + const elapsed = await timed(version); + if (n >= 2) samples[version].push(elapsed); + } + } + const stats = (values: number[]) => { + const sorted = [...values].sort((a, b) => a - b); + return { perCallMs: values.map(value => Math.round(value * 100) / 100), medianMs: sorted[Math.floor(sorted.length / 2)], + minMs: sorted[0], maxMs: sorted.at(-1) }; + }; + + const captured: Record> = { baseline: [], treatment: [] }; + for (const version of Object.keys(versions) as Array) { + const proxy = new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args), execute = selection.execute.bind(selection); + selection.execute = (async () => { captured[version].push(selection.toSQL()); return execute(); }) as typeof selection.execute; + return selection; + }) as typeof builder.from; + return builder; + }; + } }); + await versions[version](proxy as Db, sample[0]!); + } + async function plan(query: { sql: string; params: unknown[] }) { + return (await db!.$client.unsafe(`EXPLAIN (ANALYZE, BUFFERS) ${query.sql}`, query.params as never[]).values()).map(row => row[0]); + } + const counts = (await db.$client.unsafe(`select (select count(*) from heartbeat_runs)::int runs, + pg_size_pretty(pg_total_relation_size('heartbeat_runs')) runs_size, + (select count(*) from heartbeat_runs where agent_id = $1)::int runs_of_sampled_agent`, [sample[0]!.agentId]))[0]; + console.log(JSON.stringify({ baselineRevision, treatmentRevision, + postgresVersion: (await db.$client.unsafe("SELECT version()"))[0]!.version, + fixture: { ...fixture, ...counts, seedMs: Math.round(seedMs), company, synthetic: true }, + correctness: { completeOutputEqual: true, inputs: inputs.length, outcomes }, + timings: { warmups: 2, samples: iterations, inputsPerSample: sample.length, sampledAgents, order: "alternating", unit: "ms per readGitHubResponseRun call", + baseline: stats(samples.baseline), treatment: stats(samples.treatment) }, + statementsPerCall: { baseline: captured.baseline.length, treatment: captured.treatment.length }, + plans: { baseline: await Promise.all(captured.baseline.map(plan)), treatment: await Promise.all(captured.treatment.map(plan)) }, + }, null, 2)); +} finally { + try { + await db?.$client.end({ timeout: 0 }); + await temporary?.cleanup(); + if (baselineDir) await fs.rm(baselineDir, { recursive: true, force: true }); + } finally { clearInterval(keepAlive); } +} diff --git a/server/src/services/chat-channels/github/response-run-state.test.ts b/server/src/services/chat-channels/github/response-run-state.test.ts new file mode 100644 index 00000000000..c59865a1466 --- /dev/null +++ b/server/src/services/chat-channels/github/response-run-state.test.ts @@ -0,0 +1,224 @@ +import { randomUUID } from "node:crypto"; +import { and, desc, eq, or, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { chatMessageLinks, createDb, heartbeatRuns, type Db } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; +import { chatRunSourceMessageId } from "../run-origin.js"; +import { readGitHubResponseRun, responseRunColumns } from "./response-run-state.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; + +type Input = Parameters[1]; + +async function previousReadGitHubResponseRun(db: Db, input: Input) { + const [requestRun] = await db.select({ runId: heartbeatRuns.id }).from(heartbeatRuns).innerJoin(chatMessageLinks, and( + eq(chatMessageLinks.companyId, input.companyId), eq(chatMessageLinks.endpointId, input.endpointId), + eq(chatMessageLinks.deliveryId, input.deliveryId), eq(chatMessageLinks.direction, "inbound"), + or(sql`${chatMessageLinks.commentId}::text = ${heartbeatRuns.contextSnapshot}->>'wakeCommentId'`, + sql`coalesce(${heartbeatRuns.contextSnapshot}->'wakeCommentIds', '[]'::jsonb) ? ${chatMessageLinks.commentId}::text`), + )).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, input.agentId), + sql`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.contextSnapshot}->>'issueId') = ${input.issueId}`)) + .orderBy(desc(heartbeatRuns.createdAt)).limit(1); + const [latestRun] = await db.select(responseRunColumns).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, input.agentId), + sql`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.contextSnapshot}->>'issueId') = ${input.issueId}`, + )).orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); + const originMessageId = latestRun?.activeIdentityContextId ? await chatRunSourceMessageId(db, latestRun, + { allowLegacyWakeComment: true, provider: "github" }) : null; + const ownsLatestRequest = !!latestRun && (requestRun?.runId === latestRun.id || + (!!input.sourceMessageId && originMessageId === input.sourceMessageId)); + return { latestRun, ownsLatestRequest }; +} + +function random(seed: number) { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let value = Math.imul(state ^ (state >>> 15), 1 | state); + value = (value + Math.imul(value ^ (value >>> 7), 61 | value)) ^ value; + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +} + +type PlanNode = { "Relation Name"?: string; "Actual Loops"?: number; "Actual Rows"?: number; + "Rows Removed by Filter"?: number; "Rows Removed by Index Recheck"?: number; Plans?: PlanNode[] }; + +function heartbeatRunRowsRead(node: PlanNode): number { + const own = node["Relation Name"] === "heartbeat_runs" ? (node["Actual Loops"] ?? 0) + * ((node["Actual Rows"] ?? 0) + (node["Rows Removed by Filter"] ?? 0) + (node["Rows Removed by Index Recheck"] ?? 0)) : 0; + return own + (node.Plans ?? []).reduce((total, child) => total + heartbeatRunRowsRead(child), 0); +} + +type Tenant = { companyId: string; endpointId: string; agentIds: [string, string]; + requests: Array<{ issueId: string; deliveryId: string; inboundCommentId: string | null; outboundCommentId: string }> }; + +suite("GitHub response run state", () => { + let db!: Db; + let temporary: Awaited> | null = null; + const tenants: Tenant[] = []; + const history = 1_500; + + async function seedTenant(index: number): Promise { + const client = db.$client; + const [{ company }] = await client.unsafe(`insert into companies (name, issue_prefix) values ($1, $2) returning id as company`, + [`Response ${index}`, `RS${index}`]); + const agentRows = await client.unsafe(`insert into agents (company_id, name) values ($1, 'GitHub agent'), ($1, 'Other agent') + returning id`, [company]); + const agentIds = agentRows.map(row => row.id as string) as [string, string]; + const [{ application }] = await client.unsafe(`insert into tool_applications (company_id, application_key, name, type) + values ($1, 'github', 'GitHub', 'chat') returning id as application`, [company]); + const [{ connection }] = await client.unsafe(`insert into tool_connections (company_id, application_id, name, uid, transport, connection_purpose) + values ($1, $2, 'GitHub', $3, 'chat_sdk', 'channel') returning id as connection`, [company, application, `github-${index}`]); + const [{ endpoint }] = await client.unsafe(`insert into chat_endpoints (company_id, connection_id, provider, public_id, assigned_agent_id, status) + values ($1, $2, 'github', $3, $4, 'active') returning id as endpoint`, [company, connection, `response-${index}`, agentIds[0]]); + const requests: Tenant["requests"] = []; + for (let issueNumber = 1; issueNumber <= 60; issueNumber++) { + const [{ issue }] = await client.unsafe(`insert into issues (company_id, title, status) values ($1, $2, 'in_progress') returning id as issue`, + [company, `Issue ${issueNumber}`]); + const [{ conversation }] = await client.unsafe(`insert into chat_conversations (company_id, endpoint_id, issue_id, + external_conversation_id, external_label, external_thread_id) + values ($1, $2, $3, $4, $4, $4) returning id as conversation`, [company, endpoint, issue, `github:paperclipai/paperclip:${issueNumber}`]); + for (let request = 0; request < 1 + (issueNumber % 2); request++) { + const [{ delivery }] = await client.unsafe(`insert into chat_deliveries (company_id, endpoint_id, provider_event_id, + deduplication_key, event_kind, normalized_event, conversation_id, state) + values ($1, $2, $3, $3, 'message', '{}', $4, 'processed') returning id as delivery`, + [company, endpoint, `event-${issueNumber}-${request}`, conversation]); + const commentRows = await client.unsafe(`insert into issue_comments (company_id, issue_id, body) + values ($1, $2, 'Inbound'), ($1, $2, 'Outbound') returning id`, [company, issue]); + const inboundCommentId = issueNumber % 13 === 0 ? null : commentRows[0]!.id as string; + const outboundCommentId = commentRows[1]!.id as string; + await client.unsafe(`insert into chat_message_links (company_id, endpoint_id, conversation_id, delivery_id, comment_id, + provider_message_id, direction) + values ($1, $2, $3, $4, $5, $6, 'inbound'), ($1, $2, $3, $4, $7, $8, 'outbound')`, + [company, endpoint, conversation, delivery, inboundCommentId, `in-${issueNumber}-${request}`, + outboundCommentId, `out-${issueNumber}-${request}`]); + requests.push({ issueId: issue, deliveryId: delivery, inboundCommentId, outboundCommentId }); + } + } + return { companyId: company, endpointId: endpoint, agentIds, requests }; + } + + async function seedRuns(seed: number) { + const next = random(seed); + const pick = (values: readonly T[]): T => values[Math.floor(next() * values.length)]!; + const base = Date.UTC(2026, 10, 1); + const rows: Array = []; + let timeline = 0; + for (const tenant of tenants) { + const other = tenants.find(candidate => candidate !== tenant)!; + const issueIds = [...new Set(tenant.requests.map(request => request.issueId))]; + for (const issueId of issueIds) { + timeline += 1; + const requests = tenant.requests.filter(request => request.issueId === issueId); + const slots = Array.from({ length: 1 + Math.floor(next() * 6) }, (_, slot) => slot); + let linkedAt = new Set(); + for (const slot of slots) { + const tie = slot > 0 && next() < 0.3; + const minute = tie ? slot - 1 : slot; + const request = pick(requests); + let wake = pick(["none", "comment", "comments", "scalar", "sibling", "outbound", "foreign"] as const); + const links = wake === "comment" || wake === "comments" || wake === "scalar" || wake === "sibling"; + if (links && linkedAt.has(minute)) wake = "none"; + else if (links) linkedAt = new Set([...linkedAt, minute]); + const sibling = requests.find(candidate => candidate !== request) ?? request; + const foreignComment = pick(other.requests).inboundCommentId; + const wakeContext = wake === "comment" ? { wakeCommentId: request.inboundCommentId } + : wake === "comments" ? { wakeCommentIds: [randomUUID(), request.inboundCommentId] } + : wake === "scalar" ? { wakeCommentIds: request.inboundCommentId } + : wake === "outbound" ? { wakeCommentId: request.outboundCommentId } + : wake === "foreign" ? { wakeCommentId: foreignComment } + : wake === "sibling" ? { wakeCommentIds: [sibling.inboundCommentId] } : {}; + const binding = pick(["native", "native", "legacy", "legacy", "nativeElsewhere", "otherCompany", "otherAgent"] as const); + const nativeIssueId = binding === "native" ? issueId : binding === "nativeElsewhere" ? pick(issueIds) : null; + rows.push({ + id: randomUUID(), + companyId: binding === "otherCompany" ? other.companyId : tenant.companyId, + agentId: binding === "otherCompany" ? other.agentIds[0] : binding === "otherAgent" ? tenant.agentIds[1] : pick([tenant.agentIds[0], tenant.agentIds[0], tenant.agentIds[1]]), + scopeKind: nativeIssueId ? "issue" : "company", + issueId: nativeIssueId, + nativeIssueId, + status: pick(["queued", "running", "succeeded", "failed", "cancelled", "timed_out"]), + createdAt: new Date(base + timeline * 3_600_000 + minute * 60_000), + contextSnapshot: { issueId, ...wakeContext }, + }); + } + } + } + await db.insert(heartbeatRuns).values(rows); + } + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-github-response-run-state-"); + db = createDb(temporary.connectionString); + tenants.push(await seedTenant(0), await seedTenant(1)); + await seedRuns(20261114); + await db.$client.unsafe(`insert into heartbeat_runs (company_id, agent_id, status, context_snapshot, created_at) + select $1::uuid, $2::uuid, 'succeeded', jsonb_build_object('issueId', gen_random_uuid()::text, + 'prompt', (select string_agg(md5(random()::text || g::text || n::text), '') from generate_series(1, 64) g)), + timestamptz '2026-10-01' + (n || ' minutes')::interval from generate_series(1, ${history}) n`, + [tenants[0]!.companyId, tenants[0]!.agentIds[0]]); + await db.execute(sql`analyze`); + }, 120_000); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + + function inputs(): Input[] { + const all: Input[] = []; + for (const tenant of tenants) { + const other = tenants.find(candidate => candidate !== tenant)!; + for (const request of tenant.requests) { + for (const agentId of [...tenant.agentIds, other.agentIds[0]]) { + all.push({ companyId: tenant.companyId, endpointId: tenant.endpointId, agentId, issueId: request.issueId, + deliveryId: request.deliveryId, sourceMessageId: request.inboundCommentId }); + } + all.push({ companyId: tenant.companyId, endpointId: other.endpointId, agentId: tenant.agentIds[0], issueId: request.issueId, + deliveryId: request.deliveryId, sourceMessageId: null }); + all.push({ companyId: tenant.companyId, endpointId: tenant.endpointId, agentId: tenant.agentIds[0], issueId: null, + deliveryId: request.deliveryId, sourceMessageId: null }); + } + } + return all; + } + + it("returns the latest run and request ownership the previous queries returned, for every run binding and wake link", async () => { + const before = [], after = []; + for (const input of inputs()) { + before.push(await previousReadGitHubResponseRun(db, input)); + after.push(await readGitHubResponseRun(db, input)); + } + expect(after).toEqual(before); + expect(before.filter(result => result.ownsLatestRequest).length).toBeGreaterThan(20); + expect(before.filter(result => result.latestRun && !result.ownsLatestRequest).length).toBeGreaterThan(20); + expect(before.filter(result => !result.latestRun).length).toBeGreaterThan(20); + expect(new Set(before.flatMap(result => result.latestRun ? [result.latestRun.nativeIssueId ? "native" : "legacy"] : []))) + .toEqual(new Set(["native", "legacy"])); + }, 120_000); + + it("reads a few runs of the issue, not the agent's whole run history", async () => { + const tenant = tenants[0]!; + const request = tenant.requests[0]!; + const captured: Array<{ sql: string; params: unknown[] }> = []; + const capture = new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args), execute = selection.execute.bind(selection); + selection.execute = (async () => { captured.push(selection.toSQL()); return execute(); }) as typeof selection.execute; + return selection; + }) as typeof builder.from; + return builder; + }; + } }); + await readGitHubResponseRun(capture as Db, { companyId: tenant.companyId, endpointId: tenant.endpointId, + agentId: tenant.agentIds[0], issueId: request.issueId, deliveryId: request.deliveryId, sourceMessageId: null }); + expect(captured.length).toBeGreaterThan(0); + let rowsRead = 0; + for (const query of captured) { + const [row] = await db.$client.unsafe(`explain (analyze, format json) ${query.sql}`, query.params as never[]); + rowsRead += heartbeatRunRowsRead((row!["QUERY PLAN"] as Array<{ Plan: PlanNode }>)[0]!.Plan); + } + expect(rowsRead).toBeLessThan(history / 10); + }, 60_000); +}); diff --git a/server/src/services/chat-channels/github/response-run-state.ts b/server/src/services/chat-channels/github/response-run-state.ts index e0d4bc23669..8060536ddf4 100644 --- a/server/src/services/chat-channels/github/response-run-state.ts +++ b/server/src/services/chat-channels/github/response-run-state.ts @@ -1,4 +1,4 @@ -import { and, desc, eq, or, sql } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { chatMessageLinks, heartbeatRuns, type Db } from "@paperclipai/db"; import { chatRunSourceMessageId } from "../run-origin.js"; @@ -52,21 +52,30 @@ export async function readGitHubResponseRun(db: Db, input: { companyId: string; endpointId: string; agentId: string; issueId: string | null; deliveryId: string; sourceMessageId: string | null; }) { - const [requestRun] = await db.select({ runId: heartbeatRuns.id }).from(heartbeatRuns).innerJoin(chatMessageLinks, and( - eq(chatMessageLinks.companyId, input.companyId), eq(chatMessageLinks.endpointId, input.endpointId), - eq(chatMessageLinks.deliveryId, input.deliveryId), eq(chatMessageLinks.direction, "inbound"), - or(sql`${chatMessageLinks.commentId}::text = ${heartbeatRuns.contextSnapshot}->>'wakeCommentId'`, - sql`coalesce(${heartbeatRuns.contextSnapshot}->'wakeCommentIds', '[]'::jsonb) ? ${chatMessageLinks.commentId}::text`), - )).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, input.agentId), - sql`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.contextSnapshot}->>'issueId') = ${input.issueId}`)) - .orderBy(desc(heartbeatRuns.createdAt)).limit(1); - const [latestRun] = await db.select(responseRunColumns).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, input.agentId), - sql`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.contextSnapshot}->>'issueId') = ${input.issueId}`, - )).orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); - const originMessageId = latestRun?.activeIdentityContextId ? await chatRunSourceMessageId(db, latestRun, + const { issueId } = input; + if (!issueId) return { latestRun: undefined, ownsLatestRequest: false }; + const latestRunId = sql`(select latest.id from ( + (select r.id, r.created_at from ${heartbeatRuns} r + where r.company_id = ${input.companyId} and r.native_issue_id = ${issueId} and r.agent_id = ${input.agentId} + order by r.created_at desc, r.id desc limit 1) + union all + (select r.id, r.created_at from ${heartbeatRuns} r + where r.company_id = ${input.companyId} and r.native_issue_id is null + and r.context_snapshot->>'issueId' = ${issueId} and r.agent_id = ${input.agentId} + order by r.created_at desc, r.id desc limit 1) + ) latest order by latest.created_at desc, latest.id desc limit 1)`; + const wokeForRequest = sql`exists (select 1 from ${chatMessageLinks} l + where l.company_id = ${input.companyId} and l.endpoint_id = ${input.endpointId} + and l.delivery_id = ${input.deliveryId} and l.direction = 'inbound' + and (l.comment_id::text = ${heartbeatRuns.contextSnapshot}->>'wakeCommentId' + or coalesce(${heartbeatRuns.contextSnapshot}->'wakeCommentIds', '[]'::jsonb) ? l.comment_id::text))`; + const [row] = await db.select({ ...responseRunColumns, wokeForRequest }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, input.companyId), sql`${heartbeatRuns.id} = ${latestRunId}`)); + if (!row) return { latestRun: undefined, ownsLatestRequest: false }; + const { wokeForRequest: requestRunIsLatest, ...latestRun } = row; + const originMessageId = latestRun.activeIdentityContextId ? await chatRunSourceMessageId(db, latestRun, { allowLegacyWakeComment: true, provider: "github" }) : null; - const ownsLatestRequest = !!latestRun && (requestRun?.runId === latestRun.id || - (!!input.sourceMessageId && originMessageId === input.sourceMessageId)); + const ownsLatestRequest = requestRunIsLatest || + (!!input.sourceMessageId && originMessageId === input.sourceMessageId); return { latestRun, ownsLatestRequest }; }