diff --git a/server/scripts/benchmark-blocker-stall-waiters.ts b/server/scripts/benchmark-blocker-stall-waiters.ts new file mode 100644 index 00000000000..8020d1d1f4a --- /dev/null +++ b/server/scripts/benchmark-blocker-stall-waiters.ts @@ -0,0 +1,171 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { PerformanceObserver, performance, type PerformanceEntry } from "node:perf_hooks"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import v8 from "node:v8"; +import vm from "node:vm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { createDb, startEmbeddedPostgresTestDatabase } from "../../packages/db/src/index.js"; +import { markStallWakeDelivered, seedBlockerStall, seedBlockerStallCompany } from "../src/__tests__/helpers/blocker-stall-fixture.js"; +import * as current from "../src/services/recovery/blocker-stall-wakeups.js"; + +const baselineRef = process.argv.find(arg => arg.startsWith("--baseline-ref="))?.slice(15); +assert.ok(baselineRef, "Pass an explicit pre-fix --baseline-ref="); +const root = fileURLToPath(new URL("../../", import.meta.url)); +const git = (...args: string[]) => execFileSync("git", args, { cwd: root, encoding: "utf8" }); +const baselineRevision = git("rev-parse", "--verify", `${baselineRef}^{commit}`).trim(); +const modulePath = "server/src/services/recovery/blocker-stall-wakeups.ts"; +const size = (name: string, fallback: number) => Number(process.env[name] ?? fallback); +const STALLS = size("STALLS", 20), ITERATIONS = size("ITERATIONS", 300), PASSES = size("PASSES", 5); +type Wakeups = Pick; + +const keepAlive = setInterval(() => {}, 1_000); +let temporary: Awaited> | undefined; +let db: ReturnType | undefined; +let baselineDir: string | undefined; +let collections: PerformanceEntry[] = []; +const gcObserver = new PerformanceObserver(list => { collections.push(...list.getEntries()); }); +gcObserver.observe({ entryTypes: ["gc"] }); +function collectionsBetween(start: number, end: number) { + collections.push(...gcObserver.takeRecords()); + const within = collections.filter(entry => entry.startTime >= start && entry.startTime <= end); + collections = collections.filter(entry => entry.startTime > end); + return { count: within.length, ms: within.reduce((sum, entry) => sum + entry.duration, 0) }; +} +const allocated = () => (v8.getHeapStatistics() as { total_allocated_bytes?: number }).total_allocated_bytes; +v8.setFlagsFromString("--expose-gc"); +const collectGarbage = vm.runInNewContext("gc") as () => void; + +try { + const moduleUrl = new URL(`../../${modulePath}`, import.meta.url); + baselineDir = await fs.mkdtemp(path.join(root, "server/scripts/.blocker-stall-baseline-")); + const baselineFile = path.join(baselineDir, "blocker-stall-wakeups.ts"); + await fs.writeFile(baselineFile, git("show", `${baselineRevision}:${modulePath}`).replace(/from "(\.{1,2}\/[^"]+)"/g, + (_match, specifier: string) => `from ${JSON.stringify(fileURLToPath(new URL(specifier, moduleUrl)))}`)); + const baseline = await import(pathToFileURL(baselineFile).href) as Wakeups; + const versions: Array<{ label: "baseline" | "treatment"; wakeups: Wakeups }> = [ + { label: "baseline", wakeups: baseline }, { label: "treatment", wakeups: current }]; + + temporary = await startEmbeddedPostgresTestDatabase("blocker-stall-waiters-benchmark-"); + db = createDb(temporary.connectionString); + const database = db; + database.$client.options.onnotice = () => {}; + const company = await seedBlockerStallCompany(database); + const stalls = []; + for (let index = 0; index < STALLS; index++) { + const stall = await seedBlockerStall(database, company, { dependents: 1 }); + await markStallWakeDelivered(database, stall, stall.parent!.id); + await markStallWakeDelivered(database, stall, stall.dependents[0]!.id); + stalls.push(stall); + } + const enqueueWakeup = async () => { throw new Error("Every waiter of the fixture is already woken"); }; + const scenarios = { + sweep: (wakeups: Wakeups) => wakeups.wakeWaitersOfStalledIssues(database, enqueueWakeup, { companyId: company.companyId }), + task: (wakeups: Wakeups) => wakeups.wakeWaitersOfStalledIssues(database, enqueueWakeup, + { companyId: company.companyId, issueId: stalls[0]!.stalled.id }), + }; + type Scenario = keyof typeof scenarios; + const names = Object.keys(scenarios) as Scenario[]; + const waiterRead = (text: string) => text.includes("issue_blocker_stalled:"); + + let sent: Array<{ text: string; parameters: unknown[] }> | null = null; + (database.$client.options as { debug?: unknown }).debug = (_connection: number, text: string, parameters: unknown[]) => { + sent?.push({ text, parameters }); + }; + async function observe(name: Scenario, wakeups: Wakeups) { + sent = []; + const result = await scenarios[name](wakeups); + const statements = sent; + sent = null; + return { result, statements }; + } + const statements: Record = {}; + for (const name of names) { + const before = await observe(name, baseline), after = await observe(name, current); + assert.deepEqual(after, before, `${name}: result, statements and parameters must match the baseline`); + const reads = after.statements.filter(statement => waiterRead(statement.text)); + assert.ok(reads.length > 0, `${name}: the waiter read was sent`); + statements[name] = { count: after.statements.length, waiterReads: reads.length, + waiterReadBytes: Buffer.byteLength(reads[0]!.text), waiterReadParameters: reads[0]!.parameters.length }; + } + assert.equal(statements.sweep!.waiterReads, STALLS); + + const paging = await seedBlockerStallCompany(database, "paused"); + const paged = await seedBlockerStall(database, paging, { dependents: 101 }); + const orphan = await seedBlockerStall(database, paging, { parent: false, dependents: 101 }); + const shapes = { + companyPages: { companyId: paging.companyId }, + childWaiter: { companyId: paging.companyId, issueId: paged.dependents[0]!.id }, + blockerWaiter: { companyId: paging.companyId, issueId: orphan.dependents[0]!.id }, + }; + const waiterReadTexts = new Set(); + for (const [name, options] of Object.entries(shapes)) { + const observed = []; + for (const version of versions) { + sent = []; + const { retryAt, ...result } = await version.wakeups.wakeWaitersOfStalledIssues(database, enqueueWakeup, options); + observed.push({ result, deferredRetry: retryAt !== null, statements: sent }); + sent = null; + } + assert.deepEqual(observed[1], observed[0], `${name}: result, statements and parameters must match the baseline`); + for (const statement of observed[1]!.statements) if (waiterRead(statement.text)) waiterReadTexts.add(statement.text); + } + assert.equal(waiterReadTexts.size, 6, "First and later pages of a child and of a blocker stall, and one waiter of each"); + + async function measure(name: Scenario, wakeups: Wakeups) { + const written = PgDialect.prototype.sqlToQuery; + let writes = 0; + PgDialect.prototype.sqlToQuery = function (...args: Parameters) { + const query = written.apply(this, args); + if (waiterRead(query.sql)) writes += 1; + return query; + }; + try { + collectGarbage(); + const before = { cpu: process.cpuUsage(), bytes: allocated(), at: performance.now() }; + for (let index = 0; index < ITERATIONS; index++) await scenarios[name](wakeups); + const endedAt = performance.now(), cpu = process.cpuUsage(before.cpu), bytes = allocated(); + await new Promise(resolve => setImmediate(resolve)); + const gc = collectionsBetween(before.at, endedAt); + return { waiterSqlTextsWrittenPerCall: writes / ITERATIONS, cpuUsPerCall: (cpu.user + cpu.system) / ITERATIONS, + wallUsPerCall: (endedAt - before.at) * 1000 / ITERATIONS, + allocatedKbPerCall: bytes === undefined || before.bytes === undefined ? null : (bytes - before.bytes) / 1024 / ITERATIONS, + gcPausesPer1000Calls: gc.count * 1000 / ITERATIONS, gcMsPer1000Calls: gc.ms * 1000 / ITERATIONS }; + } finally { PgDialect.prototype.sqlToQuery = written; } + } + for (const version of versions) for (const name of names) await measure(name, version.wakeups); + const samples = new Map>>>(); + for (let pass = 0; pass < PASSES; pass++) { + for (const version of pass % 2 === 0 ? versions : [...versions].reverse()) { + for (const name of names) { + const key = `${name}/${version.label}`; + samples.set(key, [...samples.get(key) ?? [], await measure(name, version.wakeups)]); + } + } + } + const median = (values: number[]) => [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]!; + const results = names.flatMap(name => versions.map(version => { + const passes = samples.get(`${name}/${version.label}`)!; + const field = (key: keyof (typeof passes)[number]) => passes.every(pass => pass[key] !== null) + ? Number(median(passes.map(pass => pass[key] as number)).toFixed(2)) : null; + return { scenario: name, version: version.label, statementsPerCall: statements[name]!.count, + waiterReadsPerCall: statements[name]!.waiterReads, waiterSqlTextsWrittenPerCall: field("waiterSqlTextsWrittenPerCall"), + allocatedKbPerCall: field("allocatedKbPerCall"), cpuUsPerCall: field("cpuUsPerCall"), wallUsPerCall: field("wallUsPerCall"), + gcPausesPer1000Calls: field("gcPausesPer1000Calls"), gcMsPer1000Calls: field("gcMsPer1000Calls") }; + })); + const [{ version }] = await database.$client.unsafe("select version() as version") as unknown as [{ version: string }]; + console.log(JSON.stringify({ baselineRevision, treatmentRevision: git("rev-parse", "HEAD").trim(), + treatmentDiff: git("diff", "--stat", "HEAD", "--", modulePath).trim() || "none", + node: process.version, postgres: version.split(" ").slice(0, 2).join(" "), stalls: STALLS, iterations: ITERATIONS, passes: PASSES, + correctness: { sameResult: true, sameStatementsAndParameters: true, waiterReadShapesCompared: waiterReadTexts.size }, + waiterRead: { sqlBytes: statements.sweep!.waiterReadBytes, parameters: statements.sweep!.waiterReadParameters }, results }, null, 2)); +} finally { + gcObserver.disconnect(); + try { + await db?.$client.end({ timeout: 0 }); + await temporary?.cleanup(); + if (baselineDir) await fs.rm(baselineDir, { recursive: true, force: true }); + } finally { clearInterval(keepAlive); } +} diff --git a/server/src/__tests__/helpers/blocker-stall-fixture.ts b/server/src/__tests__/helpers/blocker-stall-fixture.ts new file mode 100644 index 00000000000..53f0bd416c0 --- /dev/null +++ b/server/src/__tests__/helpers/blocker-stall-fixture.ts @@ -0,0 +1,40 @@ +import { randomUUID } from "node:crypto"; +import { agents, agentWakeupRequests, companies, issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db"; +import { issueBlockerStalledWakeKey } from "../../services/recovery/blocker-stall-wakeups.js"; + +export type BlockerStallFixture = Awaited>; + +export async function seedBlockerStallCompany(db: Db, agentStatus: "idle" | "paused" = "idle") { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Stall", issuePrefix: `S${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Owner", role: "engineer", status: agentStatus }); + return { companyId, agentId }; +} + +export async function seedBlockerStall(db: Db, company: { companyId: string; agentId: string }, + options: { parent?: boolean; dependents?: number } = {}) { + const { companyId, agentId } = company; + const waiter = { companyId, status: "blocked" as const, assigneeAgentId: agentId }; + const [parent] = options.parent === false ? [null] + : await db.insert(issues).values({ ...waiter, title: "Parent" }).returning(); + const [stalled] = await db.insert(issues).values({ ...waiter, title: "Stalled", parentId: parent?.id ?? null }).returning(); + const dependents = options.dependents ? await db.insert(issues).values(Array.from({ length: options.dependents }, + (_, index) => ({ ...waiter, title: `Dependent ${index}` }))).returning() : []; + if (dependents.length) { + await db.insert(issueRelations).values(dependents.map(dependent => ({ companyId, issueId: stalled!.id, + relatedIssueId: dependent.id, type: "blocks" as const }))); + } + const [action] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: stalled!.id, kind: "run_recovery", + status: "escalated", ownerType: "board", cause: "execution_recovery_budget_exhausted", fingerprint: `fixture-${stalled!.id}`, + nextAction: "Retry the run or reassign the task" }).returning(); + const stall = { companyId, issueId: stalled!.id, actionId: action!.id, parentId: parent?.id ?? null }; + return { companyId, agentId, parent, stalled: stalled!, dependents, action: action!, stall }; +} + +export async function markStallWakeDelivered(db: Db, fixture: Pick, + waiterIssueId: string, status = "queued") { + await db.insert(agentWakeupRequests).values({ companyId: fixture.companyId, agentId: fixture.agentId, source: "automation", + triggerDetail: "system", reason: "issue_blocker_stalled", payload: { issueId: waiterIssueId }, status, + idempotencyKey: issueBlockerStalledWakeKey({ waiterIssueId, stalledIssueId: fixture.stall.issueId, + recoveryActionId: fixture.stall.actionId }) }); +} diff --git a/server/src/__tests__/issue-blocker-stall-waiter-statement.test.ts b/server/src/__tests__/issue-blocker-stall-waiter-statement.test.ts new file mode 100644 index 00000000000..6976610f593 --- /dev/null +++ b/server/src/__tests__/issue-blocker-stall-waiter-statement.test.ts @@ -0,0 +1,115 @@ +import { randomUUID } from "node:crypto"; +import { eq, fillPlaceholders, sql } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agentWakeupRequests, createDb, issueRelations, issues, type Db } from "@paperclipai/db"; +import { stallWaiterPageQuery, stallWaiterPageStatement, wakeWaitersOfStalledIssues } from "../services/recovery/blocker-stall-wakeups.js"; +import { markStallWakeDelivered, seedBlockerStall, seedBlockerStallCompany } from "./helpers/blocker-stall-fixture.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_RECOVERY_ACTION_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); + +(support.supported ? describe : describe.skip)("blocker stall waiter page statement", () => { + let temporary: Awaited> | undefined; + let db: Db; + type PageInput = Parameters[1]; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("blocker-stall-statement-"); + db = createDb(external ?? temporary!.connectionString); + db.$client.options.onnotice = () => {}; + }, 30_000); + afterEach(async () => { vi.restoreAllMocks(); await db.execute(sql`truncate companies cascade`); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + + const text = (value: string) => value.replace(/\s+/g, " ").trim(); + const builderQuery = ({ stall, onlyIssueId, after }: PageInput) => stallWaiterPageQuery(db, { + companyId: stall.companyId, stalledIssueId: stall.issueId, parentId: stall.parentId ?? null, hasParent: Boolean(stall.parentId), + onlyIssueId: onlyIssueId ? onlyIssueId : undefined, after: after ? after : undefined, + wakeKeySuffix: `:${stall.issueId}:${stall.actionId}`, + }); + const inputs = (stall: PageInput["stall"]): PageInput[] => [stall, { ...stall, parentId: null }].flatMap(variant => + [null, randomUUID()].flatMap(onlyIssueId => [null, randomUUID()].map(after => ({ stall: variant, onlyIssueId, after })))); + + it("sends the SQL text and the parameters that the query builder writes, for every shape of the read", () => { + const stall = { companyId: randomUUID(), issueId: randomUUID(), actionId: randomUUID(), parentId: randomUUID() }; + const texts = new Set(); + for (const input of inputs(stall)) { + const built = builderQuery(input).toSQL(); + const { statement, values } = stallWaiterPageStatement(db, input); + const prepared = statement.getQuery(); + expect(text(prepared.sql)).toBe(text(built.sql)); + expect(fillPlaceholders(prepared.params, values)).toEqual(built.params); + expect(built.params[0]).toBe(input.stall.parentId); + expect(built.params).toContain(`:${stall.issueId}:${stall.actionId}`); + texts.add(prepared.sql); + } + expect(texts.size).toBe(8); + }); + + it("returns the rows of the query builder: same values, types and order, and no other company's rows", async () => { + const company = await seedBlockerStallCompany(db), other = await seedBlockerStallCompany(db); + const f = await seedBlockerStall(db, company, { dependents: 6 }); + const orphan = await seedBlockerStall(db, company, { parent: false, dependents: 2 }); + const foreign = await seedBlockerStall(db, other, { dependents: 1 }); + const [woken, done, hidden, unassigned, ...waiting] = f.dependents; + await markStallWakeDelivered(db, f, woken!.id); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, done!.id)); + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, hidden!.id)); + await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, unassigned!.id)); + await markStallWakeDelivered(db, f, waiting[0]!.id, "failed"); + await markStallWakeDelivered(db, { ...foreign, stall: f.stall }, waiting[1]!.id); + await db.insert(issueRelations).values({ companyId: other.companyId, issueId: foreign.stalled.id, + relatedIssueId: foreign.parent!.id, type: "blocks" }); + + const both = async (input: PageInput) => { + const { statement, values } = stallWaiterPageStatement(db, input); + const prepared = await statement.execute(values), built = await builderQuery(input); + expect(prepared).toStrictEqual(built); + return prepared; + }; + const expected = [f.parent!.id, ...waiting.map(row => row.id)].sort(); + const all = await both({ stall: f.stall, onlyIssueId: null, after: null }); + expect(all.map(row => row.id)).toEqual(expected); + expect(all.find(row => row.id === f.parent!.id)).toMatchObject({ relation: "child", status: "blocked", + assigneeAgentId: company.agentId, identifier: null, responsibleUserId: null, executionWorkspaceId: null }); + expect(all.filter(row => row.id !== f.parent!.id).every(row => row.relation === "blocker")).toBe(true); + + expect((await both({ stall: f.stall, onlyIssueId: waiting[1]!.id, after: null })).map(row => row.id)).toEqual([waiting[1]!.id]); + expect(await both({ stall: f.stall, onlyIssueId: woken!.id, after: null })).toEqual([]); + expect((await both({ stall: f.stall, onlyIssueId: null, after: expected[0]! })).map(row => row.id)).toEqual(expected.slice(1)); + expect(await both({ stall: f.stall, onlyIssueId: expected[0]!, after: expected[0]! })).toEqual([]); + expect(await both({ stall: f.stall, onlyIssueId: null, after: expected.at(-1)! })).toEqual([]); + + const blockers = await both({ stall: orphan.stall, onlyIssueId: null, after: null }); + expect(blockers.map(row => [row.id, row.relation])).toEqual(orphan.dependents.map(row => row.id).sort().map(id => [id, "blocker"])); + expect((await both({ stall: orphan.stall, onlyIssueId: orphan.dependents[1]!.id, after: null })).map(row => row.id)) + .toEqual([orphan.dependents[1]!.id]); + expect((await both({ stall: orphan.stall, onlyIssueId: null, after: blockers[0]!.id })).map(row => row.id)).toEqual([blockers[1]!.id]); + expect(await both({ stall: orphan.stall, onlyIssueId: blockers[0]!.id, after: blockers[0]!.id })).toEqual([]); + + expect(await both({ stall: { ...f.stall, companyId: other.companyId }, onlyIssueId: null, after: null })).toEqual([]); + expect((await both({ stall: foreign.stall, onlyIssueId: null, after: null })).map(row => row.id).sort()) + .toEqual([foreign.parent!.id, foreign.dependents[0]!.id].sort()); + }); + + it("writes the SQL text of the waiter read one time and then only binds values", async () => { + const company = await seedBlockerStallCompany(db); + const stalls = [await seedBlockerStall(db, company), await seedBlockerStall(db, company), await seedBlockerStall(db, company)]; + for (const f of stalls) await markStallWakeDelivered(db, f, f.parent!.id); + const enqueueWakeup = vi.fn(); + await wakeWaitersOfStalledIssues(db, enqueueWakeup, { companyId: company.companyId }); + + const written = vi.spyOn(PgDialect.prototype, "sqlToQuery"); + const sent = vi.spyOn(db.$client, "unsafe"); + expect(await wakeWaitersOfStalledIssues(db, enqueueWakeup, { companyId: company.companyId })) + .toMatchObject({ stalls: 3, woken: 0, failed: 0 }); + const waiterRead = (statement: unknown) => String(statement).includes("issue_blocker_stalled:"); + expect(sent.mock.calls.filter(call => waiterRead(call[0]))).toHaveLength(3); + expect(new Set(sent.mock.calls.filter(call => waiterRead(call[0])).map(call => call[0])).size).toBe(1); + expect(written.mock.results.filter(result => waiterRead((result.value as { sql: string }).sql))).toHaveLength(0); + expect(enqueueWakeup).not.toHaveBeenCalled(); + expect(await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests)).toHaveLength(3); + }); +}); diff --git a/server/src/__tests__/prepared-statements-driver.test.ts b/server/src/__tests__/prepared-statements-driver.test.ts new file mode 100644 index 00000000000..c82520bfe8d --- /dev/null +++ b/server/src/__tests__/prepared-statements-driver.test.ts @@ -0,0 +1,69 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { createDb, type Db } from "@paperclipai/db"; +import { nativeProgressPageQuery, nativeProgressPageStatement } from "../services/chat-native-progress.js"; +import { stallWaiterPageQuery, stallWaiterPageStatement } from "../services/recovery/blocker-stall-wakeups.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); + +(support.supported ? describe : describe.skip)("prepared reads at the driver", () => { + let temporary: Awaited>; + const clients: Db[] = []; + beforeAll(async () => { temporary = await startEmbeddedPostgresTestDatabase("prepared-statements-driver-"); }, 60_000); + afterEach(async () => { + vi.restoreAllMocks(); + for (const db of clients.splice(0)) await db.$client.end({ timeout: 1 }); + }); + afterAll(async () => { await temporary?.cleanup(); }); + + const companyId = randomUUID(), issueId = randomUUID(), since = new Date(Date.now() - 3_600_000); + const stall = { companyId, issueId, actionId: randomUUID(), parentId: randomUUID() }; + const reads = [ + { name: "native chat progress page", text: "safe_native_progress_base_publication", statementName: "safe_native_chat_progress_page", + builder: (db: Db) => nativeProgressPageQuery(db, { since, companyId, issueId, cursor: null, pageSize: 200 }), + prepared: (db: Db) => { + const { statement, values } = nativeProgressPageStatement(db, { since, scope: { companyId, issueId }, cursor: null, pageSize: 200 }); + return statement.execute(values); + } }, + { name: "blocker stall waiter page", text: "issue_blocker_stalled:", statementName: "issue_blocker_stall_waiter_page", + builder: (db: Db) => stallWaiterPageQuery(db, { companyId, stalledIssueId: stall.issueId, parentId: stall.parentId, hasParent: true, + wakeKeySuffix: `:${stall.issueId}:${stall.actionId}` }), + prepared: (db: Db) => { + const { statement, values } = stallWaiterPageStatement(db, { stall, onlyIssueId: null, after: null }); + return statement.execute(values); + } }, + ]; + + it.each([ + { setting: undefined, named: false }, + { setting: false, named: false }, + { setting: true, named: true }, + ])("asks the driver for the same thing as the query builder with DATABASE_PREPARED_STATEMENTS=$setting", async ({ setting, named }) => { + const db = createDb(temporary.connectionString, { maxConnections: 1, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0, + ...(setting === undefined ? {} : { prepare: setting }) }); + clients.push(db); + const serverStatements = async (text: string) => (await db.$client.unsafe( + "select name, statement from pg_prepared_statements where statement not like '%pg_prepared_statements%'", [], { prepare: false })) + .filter(row => String(row.statement).includes(text)).map(row => String(row.name)); + for (const read of reads) { + const driver = vi.spyOn(db.$client, "unsafe"); + const sent = async (run: (db: Db) => PromiseLike) => { + driver.mockClear(); + expect(await run(db)).toEqual([]); + expect(driver).toHaveBeenCalledTimes(1); + const query = driver.mock.results[0]!.value as { options: { prepare?: boolean; simple?: boolean } }; + return { arguments: driver.mock.calls[0], options: { ...query.options }, serverStatements: await serverStatements(read.text) }; + }; + const builder = await sent(read.builder), prepared = await sent(read.prepared), repeated = await sent(read.prepared); + driver.mockRestore(); + expect(prepared).toStrictEqual(builder); + expect(repeated).toStrictEqual(builder); + expect(builder.arguments).toHaveLength(2); + expect(String(builder.arguments![0])).toContain(read.text); + expect(builder.options).toEqual({ prepare: named, simple: false }); + expect(builder.serverStatements).toHaveLength(named ? 1 : 0); + expect(builder.serverStatements.join()).not.toContain(read.statementName); + } + }); +}); diff --git a/server/src/services/recovery/blocker-stall-wakeups.ts b/server/src/services/recovery/blocker-stall-wakeups.ts index f87d1e16aa8..d8c1766e621 100644 --- a/server/src/services/recovery/blocker-stall-wakeups.ts +++ b/server/src/services/recovery/blocker-stall-wakeups.ts @@ -29,13 +29,14 @@ * reservation would still refuse that hold (remoteWorkspaceHoldRefusesStart): * its run could only fail the same way. It is woken once the hold settles. */ -import { and, desc, eq, gt, inArray, isNull, ne, notInArray, or, sql } from "drizzle-orm"; +import { and, desc, eq, gt, inArray, isNull, ne, notInArray, or, sql, type Placeholder } from "drizzle-orm"; import { agents, agentWakeupRequests, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db"; import { ISSUE_BLOCKER_STALLED_WAKE_REASON, readRemoteWorkspaceAuthority, type IssueBlockerStalledWakeContext, type IssueBlockerStallRelation } from "@paperclipai/shared"; import { logActivity } from "../activity-log.js"; import { parseHeartbeatPolicy } from "../heartbeat/timer-policy.js"; import { authorizationService } from "../authorization.js"; +import { preparedStatements } from "../prepared-statements.js"; import { readRemoteWorkspaceWriterHold, REMOTE_WORKSPACE_WRITER_PENDING_MESSAGE, remoteWorkspaceHoldRefusesStart } from "../remote-workspace-writer-hold.js"; @@ -135,6 +136,74 @@ async function readStalls(db: Db, filter: StallFilter, limit = MAX_STALLS_PER_PA type StallRow = Awaited>[number]; +type Bound = Value | Placeholder; +export type StallWaiterPageBinding = { + companyId: Bound; + stalledIssueId: Bound; + parentId: Bound; + hasParent: boolean; + onlyIssueId?: Bound; + after?: Bound; + wakeKeySuffix: Bound; +}; + +export function stallWaiterPageQuery(db: Db, bind: StallWaiterPageBinding) { + return db.select({ id: issues.id, identifier: issues.identifier, assigneeAgentId: issues.assigneeAgentId, + status: issues.status, responsibleUserId: issues.responsibleUserId, executionWorkspaceId: issues.executionWorkspaceId, + relation: sql<"child" | "blocker">`case when ${issues.id} = ${bind.parentId} then 'child' else 'blocker' end` }) + .from(issues) + .where(and( + eq(issues.companyId, bind.companyId), + ne(issues.id, bind.stalledIssueId), + inArray(issues.status, WAKEABLE_WAITER_STATUSES), + isNull(issues.hiddenAt), + sql`${issues.assigneeAgentId} is not null`, + bind.onlyIssueId !== undefined ? eq(issues.id, bind.onlyIssueId) : undefined, + bind.after !== undefined ? gt(issues.id, bind.after) : undefined, + sql`not exists (select 1 from ${agentWakeupRequests} w where w.company_id = ${bind.companyId} + and w.idempotency_key = 'issue_blocker_stalled:' || ${issues.id}::text || ${bind.wakeKeySuffix} + and w.status in (${sql.join(DELIVERED_OR_PENDING_WAKE_STATUSES.map(status => sql`${status}`), sql`, `)}))`, + or( + bind.hasParent ? eq(issues.id, bind.parentId as Bound) : sql`false`, + sql`exists (select 1 from ${issueRelations} r where r.company_id = ${bind.companyId} + and r.type = 'blocks' and r.issue_id = ${bind.stalledIssueId} and r.related_issue_id = ${issues.id})`, + ), + )).orderBy(issues.id).limit(MAX_WAITERS_PER_STALL); +} + +type StallWaiterPageShape = `${"child" | "blocker"}:${"all" | "one"}:${"first" | "after"}`; + +const stallWaiterPageStatements = preparedStatements((db, shape: StallWaiterPageShape) => { + const [parent, waiters, page] = shape.split(":"); + return stallWaiterPageQuery(db, { + companyId: sql.placeholder("companyId"), stalledIssueId: sql.placeholder("stalledIssueId"), + parentId: sql.placeholder("parentId"), hasParent: parent === "child", + onlyIssueId: waiters === "one" ? sql.placeholder("onlyIssueId") : undefined, + after: page === "after" ? sql.placeholder("after") : undefined, + wakeKeySuffix: sql.placeholder("wakeKeySuffix"), + }).prepare(`issue_blocker_stall_waiter_page_${shape.replace(/:/g, "_")}`); +}); + +type StallWaiterPageInput = { + stall: Pick; + onlyIssueId: string | null; + after: string | null; +}; + +export function stallWaiterPageStatement(db: Db, input: StallWaiterPageInput) { + const { stall } = input; + return { + statement: stallWaiterPageStatements(db, + `${stall.parentId ? "child" : "blocker"}:${input.onlyIssueId ? "one" : "all"}:${input.after ? "after" : "first"}`), + values: { + companyId: stall.companyId, stalledIssueId: stall.issueId, parentId: stall.parentId ?? null, + wakeKeySuffix: `:${stall.issueId}:${stall.actionId}`, + ...(input.onlyIssueId ? { onlyIssueId: input.onlyIssueId } : {}), + ...(input.after ? { after: input.after } : {}), + }, + }; +} + /** * Whether a run of the waiter's assignee could only fail setup on its bound * remote workspace's hold. The evidence is the assignee's own latest run on the @@ -243,29 +312,13 @@ export async function wakeWaitersOfStalledIssues(db: Db, enqueueWakeup: BlockerS let after = waiterCursors.get(cursorKey) ?? null; let scanned = 0; for (;;) { - const waiters = await db.select({ id: issues.id, identifier: issues.identifier, assigneeAgentId: issues.assigneeAgentId, - status: issues.status, responsibleUserId: issues.responsibleUserId, executionWorkspaceId: issues.executionWorkspaceId, - relation: sql<"child" | "blocker">`case when ${issues.id} = ${stall.parentId ?? null} then 'child' else 'blocker' end` }) - .from(issues) - .where(and( - eq(issues.companyId, stall.companyId), - ne(issues.id, stall.issueId), - inArray(issues.status, WAKEABLE_WAITER_STATUSES), - isNull(issues.hiddenAt), - sql`${issues.assigneeAgentId} is not null`, + const page = stallWaiterPageStatement(db, { + stall, after, onlyIssueId: // A waiter's own sweep wakes only that waiter for its children and // blockers; the stalled issue's own pass covers all of its waiters. - opts.issueId && stall.issueId !== opts.issueId ? eq(issues.id, opts.issueId) : undefined, - after ? gt(issues.id, after) : undefined, - sql`not exists (select 1 from ${agentWakeupRequests} w where w.company_id = ${stall.companyId} - and w.idempotency_key = 'issue_blocker_stalled:' || ${issues.id}::text || ${`:${stall.issueId}:${stall.actionId}`} - and w.status in (${sql.join(DELIVERED_OR_PENDING_WAKE_STATUSES.map(status => sql`${status}`), sql`, `)}))`, - or( - stall.parentId ? eq(issues.id, stall.parentId) : sql`false`, - sql`exists (select 1 from ${issueRelations} r where r.company_id = ${stall.companyId} - and r.type = 'blocks' and r.issue_id = ${stall.issueId} and r.related_issue_id = ${issues.id})`, - ), - )).orderBy(issues.id).limit(MAX_WAITERS_PER_STALL); + opts.issueId && stall.issueId !== opts.issueId ? opts.issueId : null, + }); + const waiters = await page.statement.execute(page.values); for (const waiter of waiters) { const idempotencyKey = issueBlockerStalledWakeKey({ waiterIssueId: waiter.id, stalledIssueId: stall.issueId, recoveryActionId: stall.actionId }); // (company_id, payload->>'issueId') is indexed; the key narrows to this stall.