diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index a9aad9d63fa..5c6e071c913 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -4510,9 +4510,10 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { await expect.poll(() => [...f.responseComments.values()][0]?.body, { timeout: 5000 }).toContain("stopped before completing"); await expect.poll(() => coordinator.nextWakeAt(), { timeout: 5000 }).toBeNull(); expect(f.responseWrites).toHaveLength(baseline + 1); + // The retained requests only record that they wait for a newer run. expect(await db.select().from(chatActions).where(inArray(chatActions.id, history.map(row => row.actionId)))) .toEqual(expect.arrayContaining(history.map(row => expect.objectContaining({ id: row.actionId, - status: "processed", result: { code: "fast_response_superseded" } })))); + status: "processed", result: { code: "fast_response_superseded", terminalWaitKey: expect.any(String) } })))); expect(errors).toEqual([]); } finally { await coordinator.stop(); diff --git a/server/src/services/chat-channels/github/terminal-response-work.test.ts b/server/src/services/chat-channels/github/terminal-response-work.test.ts new file mode 100644 index 00000000000..cd28076b6f2 --- /dev/null +++ b/server/src/services/chat-channels/github/terminal-response-work.test.ts @@ -0,0 +1,206 @@ +import { randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, chatActions, companies, createDb, heartbeatRuns, issues, runIdentityContexts, type Db } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; +import { createDeliveryWorkCoordinator } from "../../delivery-work-coordinator.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../delivery-work-notifications.js"; +import { idleWorkSnapshot } from "../../task-admission.js"; +import { registerGitHubTerminalResponseWork } from "./terminal-response-work.js"; + +const support = await getEmbeddedPostgresTestSupport(); + +// Staging (2026-11-15) held 1,557 processed, non-final response comments whose +// result codes were null (89%), acknowledgement_unconfirmed (10%) and +// acknowledgement_superseded (1%). Each task's latest run had ended and had +// been started by a later GitHub request, so it did not own these requests. +const ISSUES = 30; +const STALE_PER_ISSUE = 4; +const STALE = ISSUES * STALE_PER_ISSUE; +const code = (index: number) => index % 50 === 1 ? "acknowledgement_superseded" + : index % 10 === 0 ? "acknowledgement_unconfirmed" : null; + +describe.skipIf(!support.supported)("GitHub terminal response sweep backlog", () => { + let temporary: Awaited>; + let db: Db; + let coordinator: ReturnType | undefined; + const errors: unknown[] = []; + const acknowledged: string[] = []; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("github-terminal-response-work-"); + db = createDb(temporary.connectionString); + }, 60_000); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; acknowledged.length = 0; + vi.useRealTimers(); + expect(errors.splice(0)).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + + type Request = { issueId: string; deliveryId: string; actionId: string; commentId: string }; + async function seed() { + const client = db.$client; + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "GitHub backlog", issuePrefix: `G${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Reviewer" }); + const [{ application }] = await client.unsafe(`insert into tool_applications (company_id, application_key, name, type) + values ($1, 'github', 'GitHub', 'chat') returning id as application`, [companyId]); + const [{ connection }] = await client.unsafe(`insert into tool_connections (company_id, application_id, name, uid, transport, connection_purpose) + values ($1, $2, 'GitHub', $3, 'chat_sdk', 'channel') returning id as connection`, [companyId, application, `github-${companyId}`]); + const [{ endpoint }] = await client.unsafe(`insert into chat_endpoints (company_id, connection_id, provider, public_id, assigned_agent_id, status) + values ($1, $2, 'github', $3, $4, 'active') returning id as endpoint`, [companyId, connection, `backlog-${companyId}`, agentId]); + const stale: Request[] = []; + const runs: Array<{ issueId: string; runId: string }> = []; + const base = Date.UTC(2026, 9, 28); + for (let issueNumber = 1; issueNumber <= ISSUES; issueNumber++) { + const [{ issue }] = await client.unsafe(`insert into issues (company_id, title, status, assignee_agent_id) + values ($1, $2, 'in_progress', $3) returning id as issue`, [companyId, `Pull ${issueNumber}`, agentId]); + const [{ conversation }] = await client.unsafe(`insert into chat_conversations (company_id, endpoint_id, issue_id, + external_conversation_id, external_label, external_thread_id) + values ($1, $2, $3, $4, $4, $4) returning id as conversation`, [companyId, endpoint, issue, `github:paperclipai/paperclip:${issueNumber}`]); + const requests: Request[] = []; + for (let request = 0; request <= STALE_PER_ISSUE; request++) { + const [{ delivery }] = await client.unsafe(`insert into chat_deliveries (company_id, endpoint_id, provider_event_id, + deduplication_key, event_kind, normalized_event, conversation_id, state) + values ($1, $2, $3, $3, 'message', '{}', $4, 'processed') returning id as delivery`, + [companyId, endpoint, `event-${companyId}-${issueNumber}-${request}`, conversation]); + const [{ comment }] = await client.unsafe(`insert into issue_comments (company_id, issue_id, body) + values ($1, $2, 'Please review') returning id as comment`, [companyId, issue]); + await client.unsafe(`insert into chat_message_links (company_id, endpoint_id, conversation_id, delivery_id, comment_id, + provider_message_id, direction) values ($1, $2, $3, $4, $5, $6, 'inbound')`, + [companyId, endpoint, conversation, delivery, comment, `in-${companyId}-${issueNumber}-${request}`]); + const index = stale.length; + const latest = request === STALE_PER_ISSUE; + const [action] = await db.insert(chatActions).values({ companyId, endpointId: endpoint, conversationId: conversation, + deliveryId: delivery, kind: "github_response_comment", providerActionId: `github_response:${delivery}`, + payload: { agentId }, status: "processed", + result: { id: String(1_000 + index), url: `https://github.com/paperclipai/paperclip/pull/${issueNumber}#c${index}`, + final: latest, versionAt: new Date(base).toISOString(), ...(latest ? {} : { code: code(index) }) }, + updatedAt: new Date(base + index * 60_000) }).returning(); + requests.push({ issueId: issue, deliveryId: delivery, actionId: action!.id, commentId: comment }); + } + stale.push(...requests.slice(0, STALE_PER_ISSUE)); + // The latest run answered the newest request, through its identity context. + const runId = randomUUID(), contextId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issue, issueId: issue, + scopeKind: "issue", status: "succeeded", responsibleUserId: "requester", activeIdentityContextId: contextId, + contextSnapshot: { issueId: issue }, createdAt: new Date(base + issueNumber * 3_600_000) }); + await db.insert(runIdentityContexts).values({ id: contextId, companyId, runId, revision: 1, responsibleUserId: "requester", + messageId: requests[STALE_PER_ISSUE]!.commentId, cause: "chat_request", correlationId: runId, status: "accepted" }); + runs.push({ issueId: issue, runId }); + } + await db.execute(sql`analyze`); + return { companyId, agentId, endpointId: endpoint as string, stale, runs }; + } + + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, + onError: error => errors.push(error) }); + const worker = registerGitHubTerminalResponseWork(db, coordinator, async deliveryId => { + acknowledged.push(deliveryId); + // Stand in for the real acknowledgement, which records a final response. + await db.update(chatActions).set({ result: sql`${chatActions.result} || '{"final": true}'::jsonb`, updatedAt: new Date() }) + .where(and(eq(chatActions.deliveryId, deliveryId), eq(chatActions.kind, "github_response_comment"))); + }, () => true); + await worker.ready; + } + async function advance(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { + await vi.advanceTimersByTimeAsync(Math.min(1_000, until - Date.now())); await delay(1); + while (idleWorkSnapshot().active > 0) await delay(5); + } + } + /** Company-wide hints, as run lifecycle changes elsewhere in the company send them. */ + async function companyHint(companyId: string) { + await db.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatGitHubTerminalResponses, JSON.stringify({ companyId })); + }); + } + async function observe(work: () => Promise) { + const statements: string[] = [], prior = db.$client.options.debug; + db.$client.options.debug = (_connection, query) => { statements.push(query); }; + try { await work(); } finally { db.$client.options.debug = prior; } + const sweep = statements.filter(query => /"chat_actions"|heartbeat_runs|run_identity_contexts|chat_message_links/.test(query)); + return { + statements: sweep.length, + provenance: sweep.filter(query => /from "run_identity_contexts"/.test(query)).length, + }; + } + const finals = async (actionIds: string[]) => (await db.select({ result: chatActions.result }).from(chatActions) + .where(inArray(chatActions.id, actionIds))).filter(row => row.result?.final === true).length; + + it("stops re-reading processed, non-final responses that cannot progress on every company rebuild", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + const f = await seed(); + const first = await observe(async () => { await start(); await advance(5_000); }); + expect(acknowledged).toEqual([]); + const rebuilds = await observe(async () => { + for (let pass = 0; pass < 4; pass++) { await companyHint(f.companyId); await advance(31_000); } + }); + console.info(`terminal response sweep: first pass ${first.statements} statements (${first.provenance} run identity reads); ` + + `4 company rebuilds ${rebuilds.statements} statements (${rebuilds.provenance} run identity reads) for ${STALE} rows`); + // Nothing these rows wait for changed, so a rebuild reads none of them. + expect(rebuilds.provenance).toBe(0); + expect(rebuilds.statements).toBeLessThanOrEqual(4 * 2); + expect(acknowledged).toEqual([]); + // No GitHub comment changed and no row was finalized. + expect(await finals(f.stale.map(request => request.actionId))).toBe(0); + }, 300_000); + + it("still settles a waiting response once a run that owns it ends", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + const f = await seed(); + await start(); await advance(5_000); + const target = f.stale[5]!; + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, nativeIssueId: target.issueId, + issueId: target.issueId, scopeKind: "issue", status: "failed", + contextSnapshot: { issueId: target.issueId, wakeCommentId: target.commentId }, createdAt: new Date() }); + await companyHint(f.companyId); + await advance(31_000); + expect(acknowledged).toEqual([target.deliveryId]); + const rebuilds = await observe(async () => { await companyHint(f.companyId); await advance(31_000); }); + expect(rebuilds.provenance).toBe(0); + expect(acknowledged).toEqual([target.deliveryId]); + }, 300_000); + + it("still settles a response after a restart lost the hint of the reply publication it waited for", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + const f = await seed(); + const target = f.stale[9]!; + const [response] = await db.select().from(chatActions).where(eq(chatActions.id, target.actionId)); + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, nativeIssueId: target.issueId, + issueId: target.issueId, scopeKind: "issue", status: "failed", + contextSnapshot: { issueId: target.issueId, wakeCommentId: target.commentId }, createdAt: new Date() }); + // The owning run's formal review is still being published. + const [publication] = await db.insert(chatActions).values({ companyId: f.companyId, endpointId: f.endpointId, + conversationId: response!.conversationId, deliveryId: target.deliveryId, kind: "github_review_publication", + providerActionId: `github_review:${runId}`, status: "processing", + payload: { operation: "formal_review", session: { companyId: f.companyId, issueId: target.issueId, runId, agentId: f.agentId } } }) + .returning(); + await start(); await advance(5_000); + expect(acknowledged).toEqual([]); + // The publication commits, and the server stops before its lease release + // sends the targeted hint. The next start runs only a broad load. + await db.update(chatActions).set({ status: "processed", updatedAt: new Date(), + result: { id: "77", url: "https://github.com/paperclipai/paperclip/pull/1#pullrequestreview-77" } }) + .where(eq(chatActions.id, publication!.id)); + await coordinator!.stop(); coordinator = undefined; + await start(); await advance(5_000); + expect(acknowledged).toEqual([target.deliveryId]); + }, 300_000); + + it("still pauses waiting responses once their task is blocked", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + const f = await seed(); + await start(); await advance(5_000); + const blocked = f.runs[2]!.issueId; + await db.update(issues).set({ status: "blocked", updatedAt: new Date() }).where(eq(issues.id, blocked)); + await companyHint(f.companyId); + await advance(31_000); + expect(new Set(acknowledged)).toEqual(new Set(f.stale.filter(request => request.issueId === blocked).map(request => request.deliveryId))); + }, 300_000); +}); diff --git a/server/src/services/chat-channels/github/terminal-response-work.ts b/server/src/services/chat-channels/github/terminal-response-work.ts index fd60d7538e5..49c8447c25a 100644 --- a/server/src/services/chat-channels/github/terminal-response-work.ts +++ b/server/src/services/chat-channels/github/terminal-response-work.ts @@ -10,6 +10,10 @@ import { HttpError } from "../../../errors.js"; type Entry = { id: string; companyId: string; endpointId: string; deliveryId: string; issueId: string; conversationId: string; repositoryId: string | null; number: number | null; dueAt: number }; +/** A load of a whole company (or every company) rather than a targeted hint. */ +const broad = (scope?: Scope) => !scope || (!scope.issueId && !scope.deliveryId && !scope.conversationId && + !scope.endpointId && !scope.repositoryId && !scope.number); + export function registerGitHubTerminalResponseWork( db: Db, coordinator: ReturnType, @@ -19,28 +23,37 @@ export function registerGitHubTerminalResponseWork( const repositoryId = sql`${chatEndpointResources.metadata}->>'providerRepositoryId'`; const number = sql`substring(${chatConversations.externalThreadId} from '^github:[^:]+:(?:issue:)?([1-9][0-9]*)(?::rc:[1-9][0-9]*)?$')`; - // Status of the assigned agent's latest run bound to the issue, where a run's - // issue is coalesce(native_issue_id, context_snapshot->>'issueId'). The two + // The assigned agent's latest run bound to the issue, where a run's issue is + // coalesce(native_issue_id, context_snapshot->>'issueId'). The two // precedence branches are disjoint and each uses an issue index, so a // candidate does not walk (and detoast) the agent's whole run history. - const latestIssueRunStatus = sql`select latest.status from ( - (select r.status, r.created_at, r.id from ${heartbeatRuns} r + const latestIssueRun = db.select({ + status: sql`latest.status`.as("latest_run_status"), + version: sql`latest.id::text || ':' || latest.status || ':' || extract(epoch from latest.updated_at)::text` + .as("latest_run_version"), + }).from(sql`((select r.status, r.created_at, r.id, r.updated_at from ${heartbeatRuns} r where r.company_id = ${chatActions.companyId} and r.native_issue_id = ${chatConversations.issueId} and r.agent_id = ${chatEndpoints.assignedAgentId} order by r.created_at desc, r.id desc limit 1) union all - (select r.status, r.created_at, r.id from ${heartbeatRuns} r + (select r.status, r.created_at, r.id, r.updated_at from ${heartbeatRuns} r where r.company_id = ${chatActions.companyId} and r.native_issue_id is null and r.context_snapshot->>'issueId' = ${chatConversations.issueId}::text and r.agent_id = ${chatEndpoints.assignedAgentId} - order by r.created_at desc, r.id desc limit 1) - ) latest order by latest.created_at desc, latest.id desc limit 1`; + order by r.created_at desc, r.id desc limit 1)) latest`) + .orderBy(sql`latest.created_at desc, latest.id desc`).limit(1).as("latest_issue_run"); + // A request whose latest run does not own it (or whose pause is already + // shown) can progress only once the task's latest run, the task's status or + // the request's own row changes. Such a request records this key, and a broad + // rebuild skips it until one of them changes. A targeted load still reads it. + const actionVersion = sql`extract(epoch from ${chatActions.updatedAt})::text`; + const waitKey = sql`coalesce(${latestIssueRun.version}, 'none') || '|' || ${issues.status} || '|' || ${actionVersion}`; async function page(scope?: Scope, cursor?: string, actionId?: string) { return db.select({ id: chatActions.id, companyId: chatActions.companyId, endpointId: chatActions.endpointId, deliveryId: chatActions.deliveryId, conversationId: chatConversations.id, issueId: issues.id, agentId: chatEndpoints.assignedAgentId, issueStatus: issues.status, - deliveryCreatedAt: chatDeliveries.createdAt, repositoryId, number, result: chatActions.result, + deliveryCreatedAt: chatDeliveries.createdAt, repositoryId, number, result: chatActions.result, actionVersion, waitKey, sourceMessageId: sql`(select l.comment_id from ${chatMessageLinks} l where l.company_id = ${chatActions.companyId} and l.endpoint_id = ${chatActions.endpointId} and l.delivery_id = ${chatActions.deliveryId} and l.direction = 'inbound' limit 1)`, @@ -53,10 +66,12 @@ export function registerGitHubTerminalResponseWork( .innerJoin(chatDeliveries, and(eq(chatDeliveries.id, chatActions.deliveryId), eq(chatDeliveries.companyId, chatActions.companyId))) .innerJoin(issues, and(eq(issues.id, chatConversations.issueId), eq(issues.companyId, chatActions.companyId))) .leftJoin(chatEndpointResources, and(eq(chatEndpointResources.id, chatConversations.resourceId), eq(chatEndpointResources.companyId, chatActions.companyId))) + .leftJoinLateral(latestIssueRun, sql`true`) .where(and(eq(chatActions.kind, "github_response_comment"), eq(chatActions.status, "processed"), isNotNull(chatActions.deliveryId), isNotNull(chatEndpoints.assignedAgentId), sql`coalesce(${chatActions.result}->>'final', 'false') <> 'true'`, - sql`(${issues.status} = 'blocked' or (${latestIssueRunStatus}) in ('succeeded', 'failed', 'interrupted', 'timed_out', 'cancelled'))`, + sql`(${issues.status} = 'blocked' or ${latestIssueRun.status} in ('succeeded', 'failed', 'interrupted', 'timed_out', 'cancelled'))`, + broad(scope) && !actionId ? sql`${chatActions.result}->>'terminalWaitKey' is distinct from ${waitKey}` : undefined, scope ? eq(chatActions.companyId, scope.companyId) : undefined, scope?.issueId ? eq(issues.id, scope.issueId) : undefined, scope?.conversationId ? eq(chatActions.conversationId, scope.conversationId) : undefined, @@ -67,14 +82,20 @@ export function registerGitHubTerminalResponseWork( cursor ? gt(chatActions.id, cursor) : undefined, actionId ? eq(chatActions.id, actionId) : undefined, )).orderBy(asc(chatActions.id)).limit(100); } - async function entry(row: Awaited>[number]): Promise { + type Row = Awaited>[number]; + /** The request's entry when it is owed now, else whether its wait may be + * recorded. A wait on the owning run's reply or presentation is never + * recorded: a publication that commits it changes no wait-key input, and a + * restart can lose the hint that its lease release sends. */ + async function entry(row: Row): Promise { const pending: Entry = { id: row.id, companyId: row.companyId, endpointId: row.endpointId, deliveryId: row.deliveryId!, issueId: row.issueId, conversationId: row.conversationId, repositoryId: row.repositoryId, number: row.number ? Number(row.number) : null, dueAt: row.leaseExpiresAt ? Date.parse(row.leaseExpiresAt) : 0 }; try { const execution = await githubResponseExecution(db, { ...row, agentId: row.agentId!, deliveryId: row.deliveryId! }); if (execution.body === undefined || execution.reply === "unsettled" || - (execution.paused && row.result?.pausedVersion === execution.version)) return null; + (execution.paused && row.result?.pausedVersion === execution.version)) + return { parkable: !execution.presentationPending && execution.reply === "none" }; } catch (error) { if (!(error instanceof HttpError) || error.status !== 403) throw error; // Keep this request eligible for the acknowledgement owner's authority @@ -82,6 +103,27 @@ export function registerGitHubTerminalResponseWork( } return pending; } + /** Record the wait of requests that cannot progress yet. A request whose row + * changed after the read gets no key and is read again. The write leaves + * updated_at alone, because the key includes it. */ + async function park(rows: Row[]) { + const waiting = rows.filter(row => row.result?.terminalWaitKey !== row.waitKey); + if (!waiting.length) return; + await db.execute(sql`update ${chatActions} + set result = coalesce(${chatActions.result}, '{}'::jsonb) || jsonb_build_object('terminalWaitKey', waiting.key) + from unnest(${sql.param(waiting.map(row => row.id))}::uuid[], ${sql.param(waiting.map(row => row.actionVersion))}::text[], + ${sql.param(waiting.map(row => row.waitKey))}::text[]) as waiting(id, version, key) + where ${chatActions.id} = waiting.id and ${actionVersion} = waiting.version + and ${chatActions.status} = 'processed' and coalesce(${chatActions.result}->>'final', 'false') <> 'true'`); + } + async function current(actionId: string) { + const [row] = await page(undefined, undefined, actionId); + if (!row) return null; + const value = await entry(row); + if (!("parkable" in value)) return value; + if (value.parkable) await park([row]); + return null; + } return registerScopedDeadlineWork(coordinator, { queue: DELIVERY_QUEUES.chatGitHubTerminalResponses, @@ -97,22 +139,23 @@ export function registerGitHubTerminalResponseWork( let cursor: string | undefined; while (canRun() && !signal?.aborted) { const rows = await page(scope, cursor); + const waiting: Row[] = []; for (const row of rows) { if (!canRun() || signal?.aborted) throw signal?.reason ?? new Error("GitHub terminal-response recovery stopped"); - const value = await entry(row); if (value) entries.push(value); + const value = await entry(row); + if (!("parkable" in value)) entries.push(value); else if (value.parkable) waiting.push(row); } + await park(waiting); if (rows.length < 100) return entries; cursor = rows[rows.length - 1].id; } throw signal?.reason ?? new Error("GitHub terminal-response recovery stopped"); }, check: async (value: Entry, signal?: AbortSignal) => { - const [row] = await page(undefined, undefined, value.id); - const current = row ? await entry(row) : null; - if (!current || current.dueAt > Date.now() || !canRun() || signal?.aborted) return current; - await acknowledge(current.deliveryId, () => canRun() && !signal?.aborted); - const [remaining] = await page(undefined, undefined, value.id); - return remaining ? entry(remaining) : null; + const due = await current(value.id); + if (!due || due.dueAt > Date.now() || !canRun() || signal?.aborted) return due; + await acknowledge(due.deliveryId, () => canRun() && !signal?.aborted); + return current(value.id); }, }, canRun); }