diff --git a/scripts/benchmark-chat-reply-lineage.ts b/scripts/benchmark-chat-reply-lineage.ts new file mode 100644 index 00000000000..353b92de5be --- /dev/null +++ b/scripts/benchmark-chat-reply-lineage.ts @@ -0,0 +1,110 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { promises as fs, readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { applyPendingMigrations, createDb, startEmbeddedPostgresTestDatabase, type Db } from "../packages/db/src/index.js"; +import { resolveChatOriginPublicationBindings } from "../server/src/services/issues.js"; +import { captureLineageReads, chatReplyLineageFixture, cleanupChatReplyLineageFixture, expectedLineageContext, type LineageQuery, type LineageResolver } from "../server/src/__tests__/helpers/chat-reply-lineage.js"; +import { resolveExternalTestDatabaseUrl } from "../server/src/__tests__/helpers/external-test-database.js"; + +const repoRoot = fileURLToPath(new URL("../", import.meta.url)); +const ref = process.argv.find(arg => arg.startsWith("--baseline-ref="))?.slice(15); +assert.ok(ref, "Pass a pinned pre-fix --baseline-ref"); +const git = (args: string[]) => execFileSync("git", args, { cwd: repoRoot, encoding: "utf8" }).trim(); +const sourcePath = "server/src/services/issues.ts"; +const baselineRevision = git(["rev-parse", `${ref}^{commit}`]), treatmentRevision = git(["rev-parse", "HEAD"]); +const uncommittedChanges = git(["status", "--porcelain"]), treatmentDiff = git(["diff", "HEAD", "--", sourcePath]); +const sourceHash = () => createHash("sha256").update(readFileSync(path.join(repoRoot, sourcePath))).digest("hex"); +const treatmentSourceSha256 = sourceHash(); +const keepAlive = setInterval(() => {}, 1000); +let db: Db | undefined, temporary: Awaited> | undefined, baselineDirectory: string | undefined; +const companyIds: string[] = []; +let benchmarkFailure: { error: unknown } | undefined; +try { + const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_CHAT_LINEAGE_BENCHMARK_DATABASE_URL"); + if (externalUrl) await applyPendingMigrations(externalUrl); + else temporary = await startEmbeddedPostgresTestDatabase("chat-lineage-benchmark-"); + db = createDb(externalUrl ?? temporary!.connectionString); + baselineDirectory = await fs.mkdtemp(path.join(repoRoot, "server/scripts/.chat-lineage-baseline-")); + const original = pathToFileURL(path.join(repoRoot, sourcePath)); + const baselineSource = execFileSync("git", ["show", `${baselineRevision}:${sourcePath}`], { cwd: repoRoot, encoding: "utf8" }); + await fs.writeFile(path.join(baselineDirectory, "issues.ts"), baselineSource.replace(/(from\s+|import\()(["'])(\.{1,2}\/[^"']+)\2/g, + (_match, prefix: string, quote: string, specifier: string) => `${prefix}${quote}${fileURLToPath(new URL(specifier, original))}${quote}`)); + const baseline = (await import(pathToFileURL(path.join(baselineDirectory, "issues.ts")).href)).resolveChatOriginPublicationBindings as LineageResolver; + const measurements = []; + for (const scenario of [{ name: "small", bytes: 16, hops: 0 }, { name: "wide", bytes: 192 * 1024, hops: 0 }, { name: "wide-24-continuations", bytes: 192 * 1024, hops: 24 }]) { + const f = await chatReplyLineageFixture(db, scenario.bytes); companyIds.push(f.companyId); + let runId = f.root.id; + for (let hop = 0; hop < scenario.hops; hop++) runId = (await f.continueFrom(runId)).run.id; + await db.$client.unsafe("ANALYZE"); + async function capture(resolver: LineageResolver) { + const reads: LineageQuery[] = []; + const result = await resolver(captureLineageReads(db!, reads), f.companyId, f.issueId, runId); + assert.deepEqual(result, f.expected); + assert.equal(reads.length, scenario.hops + 1); + return { result, reads }; + } + const before = await capture(baseline), after = await capture(resolveChatOriginPublicationBindings); + assert.deepEqual(after.result, before.result); + for (let index = 0; index < before.reads.length; index++) { + const row = before.reads[index].rows[0]; + assert.deepEqual(after.reads[index].rows, [{ ...row, contextSnapshot: expectedLineageContext(row.contextSnapshot) }]); + } + const variants = [before, after], samples: number[][] = [[], []], bytes: number[] = [0, 0]; + for (let iteration = 0; iteration < 9; iteration++) { + for (const variant of iteration % 2 ? [1, 0] : [0, 1]) { + const started = performance.now(); let returned = 0; + for (const query of variants[variant].reads) { + const rows = await db.$client.unsafe(query.sql, query.params as never[]).raw(); + returned += rows.reduce((sum, row) => sum + 7 + row.reduce((n: number, value: Buffer | null) => n + 4 + (value?.length ?? 0), 0), 0); + } + if (iteration >= 2) samples[variant].push(performance.now() - started); + bytes[variant] = returned; + } + } + const minimumSaving = Buffer.byteLength(f.context.prompt) * (scenario.hops + 1); + assert.ok(bytes[0] - bytes[1] >= minimumSaving, "All unrelated synthetic body bytes must be removed before transfer"); + const plans = []; + for (const variant of variants) { + const query = variant.reads[0]; + const [plan] = await db.$client.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${query.sql}`, query.params as never[]); + plans.push(plan["QUERY PLAN"]); + } + const timing = (values: number[]) => { const sorted = [...values].sort((a, b) => a - b); return { medianMs: sorted[3], minMs: sorted[0], maxMs: sorted[6], samplesMs: values }; }; + measurements.push({ scenario, bodyBytesPerRun: Buffer.byteLength(f.context.prompt), completeBindingsEqual: true, selectedColumnsEqual: true, + baselineDataRowBytes: bytes[0], treatmentDataRowBytes: bytes[1], minimumSaving, baselineTiming: timing(samples[0]), treatmentTiming: timing(samples[1]), + baselineQueries: before.reads.map(({ rows: _rows, ...query }) => query), treatmentQueries: after.reads.map(({ rows: _rows, ...query }) => query), plans }); + } + assert.equal(sourceHash(), treatmentSourceSha256, "Measured service source changed during the benchmark"); + console.log(JSON.stringify({ baselineRevision, treatmentRevision, sourcePath, treatmentSourceSha256, uncommittedChanges, treatmentDiff, + byteBoundary: "Exact text-format PostgreSQL DataRow frames; excludes row descriptions, command metadata, TLS and TCP. Timings cover captured lineage SQL only, not the complete authorization service.", + dependencyScope: "Only the pinned baseline issues.ts is loaded from history; imported helpers and synthetic fixtures are held identical.", + postgres: await db.$client.unsafe("select version()"), settings: await db.$client.unsafe("select name,setting,unit from pg_settings where name in ('shared_buffers','work_mem','jit') order by name"), + node: process.version, measurements }, null, 2)); +} catch (error) { + benchmarkFailure = { error }; + throw error; +} finally { + try { + try { + if (db) for (const companyId of companyIds) await cleanupChatReplyLineageFixture(db, companyId); + } finally { + try { + await db?.$client.end({ timeout: 0 }); + } finally { + try { + await temporary?.cleanup(); + } finally { + try { + if (baselineDirectory) await fs.rm(baselineDirectory, { recursive: true, force: true }); + } finally { clearInterval(keepAlive); } + } + } + } + } catch (error) { + if (benchmarkFailure) throw new AggregateError([benchmarkFailure.error, error], "Benchmark and cleanup failed"); + throw error; + } +} diff --git a/server/src/__tests__/helpers/chat-reply-lineage.ts b/server/src/__tests__/helpers/chat-reply-lineage.ts new file mode 100644 index 00000000000..5a5b65baedc --- /dev/null +++ b/server/src/__tests__/helpers/chat-reply-lineage.ts @@ -0,0 +1,67 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { agents, agentWakeupRequests, chatConversations, chatEndpoints, chatMessageLinks, companies, heartbeatRuns, issueComments, issues, issueThreadInteractions, toolApplications, toolConnections, toolActionRequests, toolActionDeliveries, toolInvocations, type Db } from "@paperclipai/db"; +import { resolveChatOriginPublicationBindings } from "../../services/issues.js"; + +export const lineageContextKeys = ["issueId", "taskId", "wakeCommentIds", "wakeCommentId", "commentId", "source", "sourceRunId", "interactionId", "endpointId"] as const; +export type LineageResolver = typeof resolveChatOriginPublicationBindings; +export type LineageQuery = { sql: string; params: unknown[]; rows: Array> }; + +export function captureLineageReads(db: Db, queries: LineageQuery[]): Db { + return new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const query = from(...args), execute = query.execute.bind(query); + if (args[0] === heartbeatRuns) query.execute = async (...executeArgs: Parameters) => { + const statement = query.toSQL(); + const rows = await execute(...executeArgs); + if (fields[0] && Object.keys(fields[0]).join(",") === "agentId,responsibleUserId,contextSnapshot") queries.push({ ...statement, rows }); + return rows; + }; + return query; + }) as typeof builder.from; + return builder; + }; + } }); +} + +export function expectedLineageContext(value: unknown): unknown { + if (value === null || typeof value !== "object" || Array.isArray(value)) return value; + return Object.fromEntries(lineageContextKeys.filter(key => Object.hasOwn(value, key)).map(key => [key, (value as Record)[key]])); +} + +export async function chatReplyLineageFixture(db: Db, bytes = 192 * 1024) { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Synthetic reply lineage", issuePrefix: `L${companyId.replaceAll("-", "").slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Synthetic worker", adapterType: "paperclip_runner" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Synthetic external reply", status: "in_progress", assigneeAgentId: agentId }); + const [application] = await db.insert(toolApplications).values({ companyId, name: "Synthetic chat", applicationKey: randomUUID(), type: "chat" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: application.id, name: "Synthetic chat", uid: randomUUID(), transport: "chat_sdk", connectionPurpose: "channel" }).returning(); + const [endpoint] = await db.insert(chatEndpoints).values({ companyId, connectionId: connection.id, assignedAgentId: agentId, provider: "slack", publicId: randomUUID(), status: "active" }).returning(); + const [conversation] = await db.insert(chatConversations).values({ companyId, endpointId: endpoint.id, issueId, externalConversationId: randomUUID(), externalThreadId: randomUUID(), externalLabel: "Synthetic conversation" }).returning(); + const [comment] = await db.insert(issueComments).values({ companyId, issueId, authorType: "user", authorUserId: "synthetic-requester", body: "Please continue" }).returning(); + await db.insert(chatMessageLinks).values({ companyId, endpointId: endpoint.id, conversationId: conversation.id, commentId: comment.id, providerMessageId: randomUUID(), direction: "inbound" }); + const context = { source: "chat:slack", issueId, endpointId: endpoint.id, wakeCommentIds: [comment.id], prompt: randomBytes(bytes).toString("base64") }; + const insertRun = async (snapshot: unknown = context, overrides: Partial = {}) => { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded", contextSnapshot: snapshot as never, ...overrides }).returning(); + return run; + }; + const root = await insertRun(); + async function continueFrom(sourceRunId: string) { + const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId, sourceRunId, kind: "request_confirmation", status: "accepted", payload: { version: 1 } as never }).returning(); + const run = await insertRun({ issueId, source: "issue.interaction.accept", sourceRunId, interactionId: interaction.id, prompt: context.prompt }); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "automation", reason: "issue_commented", status: "claimed", runId: run.id, idempotencyKey: `interaction:${interaction.id}:accepted` }).returning(); + return { run, interaction, wake }; + } + return { companyId, agentId, issueId, endpoint, conversation, comment, context, root, insertRun, continueFrom, + expected: [{ companyId, endpointId: endpoint.id, conversationId: conversation.id }] }; +} + +export async function cleanupChatReplyLineageFixture(db: Db, companyId: string) { + for (const table of [agentWakeupRequests, toolActionDeliveries, toolActionRequests, toolInvocations, issueThreadInteractions, chatMessageLinks, chatConversations, chatEndpoints, heartbeatRuns, issueComments, issues, toolConnections, toolApplications, agents]) { + await db.delete(table).where(eq(table.companyId, companyId)); + } + await db.delete(companies).where(eq(companies.id, companyId)); +} diff --git a/server/src/services/chat-reply-lineage-context.test.ts b/server/src/services/chat-reply-lineage-context.test.ts new file mode 100644 index 00000000000..41ad66d3f3e --- /dev/null +++ b/server/src/services/chat-reply-lineage-context.test.ts @@ -0,0 +1,111 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { applyPendingMigrations, createDb, heartbeatRuns, agentWakeupRequests, toolInvocations, toolActionRequests, toolActionDeliveries } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js"; +import { captureLineageReads, chatReplyLineageFixture, cleanupChatReplyLineageFixture, expectedLineageContext, type LineageQuery } from "../__tests__/helpers/chat-reply-lineage.js"; +import { resolveChatOriginPublicationBindings } from "./issues.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_TEST_DATABASE_URL"); +const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("chat reply lineage context projection", () => { + let db: ReturnType; + let temporary: Awaited> | undefined; + const companyIds: string[] = []; + async function fixture(bytes?: number) { const f = await chatReplyLineageFixture(db, bytes); companyIds.push(f.companyId); return f; } + beforeAll(async () => { + if (externalUrl) await applyPendingMigrations(externalUrl); + else temporary = await startEmbeddedPostgresTestDatabase("chat-lineage-projection-"); + db = createDb(externalUrl ?? temporary!.connectionString); + }, 90_000); + afterAll(async () => { + try { + if (db) for (const companyId of companyIds) await cleanupChatReplyLineageFixture(db, companyId); + } finally { + try { await db?.$client.end({ timeout: 0 }); } + finally { await temporary?.cleanup(); } + } + }); + + it("projects the complete routing context before a wide body crosses PostgreSQL", async () => { + const f = await fixture(), reads: LineageQuery[] = []; + expect(await resolveChatOriginPublicationBindings(captureLineageReads(db, reads), f.companyId, f.issueId, f.root.id)).toEqual(f.expected); + expect(reads).toHaveLength(1); + const rows = await db.$client.unsafe(reads[0].sql, reads[0].params as never[]).raw(); + expect(rows.reduce((total, row) => total + row.reduce((n: number, value: Buffer | null) => n + (value?.length ?? 0), 0), 0)).toBeLessThan(1024); + expect(reads[0].rows).toEqual([{ agentId: f.agentId, responsibleUserId: null, contextSnapshot: expectedLineageContext(f.context) }]); + }); + + it.each([null, false, true, 0, 12, "", "malformed", [], ["source", { source: "chat:slack" }], {}, + { issueId: null, taskId: false, source: 3, interactionId: null, wakeCommentIds: { id: "not-array" } }, + ])("preserves malformed JSON and key absence for %j", async snapshot => { + const f = await fixture(1), run = await f.insertRun(snapshot), reads: LineageQuery[] = []; + expect(await resolveChatOriginPublicationBindings(captureLineageReads(db, reads), f.companyId, f.issueId, run.id)).toEqual([]); + expect(reads[0].rows[0].contextSnapshot).toEqual(expectedLineageContext(snapshot)); + }); + + it("keeps all nine keys and their JSON values, including explicit nulls and arrays", async () => { + const f = await fixture(1); + const snapshot = { issueId: null, taskId: f.issueId, wakeCommentIds: [f.comment.id, 1, null, f.comment.id], wakeCommentId: f.comment.id, commentId: f.comment.id, + source: "chat:slack", sourceRunId: { invalid: true }, interactionId: null, endpointId: f.endpoint.id, omitted: "wide unrelated value" }; + const run = await f.insertRun(snapshot), reads: LineageQuery[] = []; + expect(await resolveChatOriginPublicationBindings(captureLineageReads(db, reads), f.companyId, f.issueId, run.id)).toEqual(f.expected); + expect(reads[0].rows[0].contextSnapshot).toEqual(expectedLineageContext(snapshot)); + expect(Object.hasOwn(reads[0].rows[0].contextSnapshot as object, "interactionId")).toBe(true); + expect(Object.hasOwn(reads[0].rows[0].contextSnapshot as object, "omitted")).toBe(false); + }); + + it("preserves explicit JSON null as well as SQL null", async () => { + const f = await fixture(1), run = await f.insertRun(null), reads: LineageQuery[] = []; + await db.$client.unsafe("update heartbeat_runs set context_snapshot = 'null'::jsonb where id = $1::uuid", [run.id]); + expect(await resolveChatOriginPublicationBindings(captureLineageReads(db, reads), f.companyId, f.issueId, run.id)).toEqual([]); + expect(reads[0].rows[0].contextSnapshot).toBeNull(); + }); + + it.each([ + { name: "missing", snapshot: {}, wake: {}, allowed: true }, + { name: "explicit null", snapshot: { interactionId: null }, wake: { interactionId: null }, allowed: true }, + { name: "missing versus null", snapshot: { interactionId: null }, wake: {}, allowed: false }, + { name: "equal strings", snapshot: { interactionId: "synthetic-interaction" }, wake: { interactionId: "synthetic-interaction" }, allowed: true }, + { name: "number versus text", snapshot: { interactionId: 12 }, wake: { interactionId: "12" }, allowed: false }, + { name: "separate objects", snapshot: { interactionId: { id: "same" } }, wake: { interactionId: { id: "same" } }, allowed: false }, + ])("retains strict tool-action interaction semantics for $name and requires the durable receipt", async ({ snapshot, wake, allowed }) => { + const f = await fixture(1), continued = await f.continueFrom(f.root.id); + const [invocation] = await db.insert(toolInvocations).values({ companyId: f.companyId, issueId: f.issueId, agentId: f.agentId, runId: f.root.id, toolName: "synthetic-operation" }).returning(); + const [request] = await db.insert(toolActionRequests).values({ companyId: f.companyId, issueId: f.issueId, invocationId: invocation.id, + interactionId: continued.interaction.id, requestedByAgentId: f.agentId, status: "executed", canonicalArgumentsHash: "synthetic", canonicalArgumentsSummary: { summary: "Synthetic operation" } }).returning(); + await db.insert(toolActionDeliveries).values({ companyId: f.companyId, issueId: f.issueId, actionRequestId: request.id, interactionId: continued.interaction.id }); + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId, source: "tool_action_review", sourceRunId: f.root.id, ...snapshot } }).where(eq(heartbeatRuns.id, continued.run.id)); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", reason: "issue_commented", status: "claimed", runId: continued.run.id, + idempotencyKey: `tool-action-response:${request.id}`, payload: { sourceRunId: f.root.id, toolActionRequestIds: [request.id], ...wake } }); + expect(await resolveChatOriginPublicationBindings(db, f.companyId, f.issueId, continued.run.id)).toEqual(allowed ? f.expected : []); + await db.delete(toolActionDeliveries).where(eq(toolActionDeliveries.actionRequestId, request.id)); + expect(await resolveChatOriginPublicationBindings(db, f.companyId, f.issueId, continued.run.id)).toEqual([]); + }); + + it("retains valid lineage beyond sixteen hops and rejects a durable cycle", async () => { + const f = await fixture(1); + let previous = f.root.id; + for (let index = 0; index < 24; index++) previous = (await f.continueFrom(previous)).run.id; + expect(await resolveChatOriginPublicationBindings(db, f.companyId, f.issueId, previous)).toEqual(f.expected); + const cycle = await f.continueFrom(previous); + await db.update(heartbeatRuns).set({ contextSnapshot: { source: "issue.interaction.accept", issueId: f.issueId, interactionId: cycle.interaction.id, sourceRunId: previous } }).where(eq(heartbeatRuns.id, f.root.id)); + await db.update(agentWakeupRequests).set({ runId: f.root.id }).where(eq(agentWakeupRequests.id, cycle.wake.id)); + const reads: LineageQuery[] = []; + expect(await resolveChatOriginPublicationBindings(captureLineageReads(db, reads), f.companyId, f.issueId, previous)).toEqual([]); + expect(reads).toHaveLength(25); + }); + + it("rejects mismatched company, task, agent, and endpoint bindings", async () => { + const f = await fixture(1), other = await fixture(1); + expect(await resolveChatOriginPublicationBindings(db, other.companyId, f.issueId, f.root.id)).toEqual([]); + for (const change of [{ issueId: other.issueId }, { endpointId: other.endpoint.id }, { source: false }, { wakeCommentIds: [randomUUID()], wakeCommentId: 12 }]) { + const run = await f.insertRun({ ...f.context, ...change }); + expect(await resolveChatOriginPublicationBindings(db, f.companyId, f.issueId, run.id)).toEqual([]); + } + const continued = await f.continueFrom(f.root.id); + await db.update(heartbeatRuns).set({ agentId: other.agentId }).where(eq(heartbeatRuns.id, continued.run.id)); + expect(await resolveChatOriginPublicationBindings(db, f.companyId, f.issueId, continued.run.id)).toEqual([]); + }); +}); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index c6e54798555..391d08c660f 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1170,7 +1170,13 @@ export async function resolveChatOriginPublicationBindings( .select({ agentId: heartbeatRuns.agentId, responsibleUserId: heartbeatRuns.responsibleUserId, - contextSnapshot: heartbeatRuns.contextSnapshot, + contextSnapshot: sql | null>` + case when jsonb_typeof(${heartbeatRuns.contextSnapshot}) = 'object' then + (select coalesce(jsonb_object_agg(entry.key, entry.value), '{}'::jsonb) + from jsonb_each(${heartbeatRuns.contextSnapshot}) as entry + where entry.key in ('issueId', 'taskId', 'wakeCommentIds', 'wakeCommentId', + 'commentId', 'source', 'sourceRunId', 'interactionId', 'endpointId')) + else ${heartbeatRuns.contextSnapshot} end`, }) .from(heartbeatRuns) .where(