diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 707beb57534..fa0fea2c7f6 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -81,7 +81,7 @@ import { GITHUB_INLINE_RETRY_WINDOW_MS } from "../services/chat-channels/github/ import { githubChatReviewService } from "../services/chat-github-reviews.js"; import { resolveGitHubReviewInstructions } from "../services/chat-channels/github/review-instructions.js"; import { githubResponseCommentService } from "../services/chat-github-response-comments.js"; -import { githubReviewCheckService } from "../services/chat-github-checks.js"; +import { GITHUB_REVIEW_CHECK_MAX_HEAD_POLL_MS, githubReviewCheckService } from "../services/chat-github-checks.js"; import { githubAutomaticReviewEvent } from "../services/chat-github-events.js"; import { githubBotToolsForSession, syncGitHubBotTools } from "../services/chat-github-tools.js"; import { buildPaperclipRuntimeMcpServers } from "../services/heartbeat/run-preparation.js"; @@ -7740,6 +7740,81 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { } finally { release(); await coordinator.stop(); } }); + async function unresolvedReviewCheckPasses() { + const f = await reviewCheckWorkerFixture("running"); + await db.update(chatGitHubReviews).set({ runId: null }).where(eq(chatGitHubReviews.id, f.review.id)); + const [delivery] = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, f.endpoint.id)); + await db.update(chatDeliveries).set({ normalizedEvent: { ...delivery.normalizedEvent, + githubAutomatic: { context: { deliveryId: f.event.deliveryId } } } }).where(eq(chatDeliveries.id, delivery.id)); + await db.update(heartbeatRuns).set({ contextSnapshot: { wakeCommentId: randomUUID() } }).where(eq(heartbeatRuns.id, f.run.id)); + await f.enqueue(); + let clock = Date.now(); + const now = vi.spyOn(Date, "now").mockImplementation(() => clock); + onTestFinished(() => now.mockRestore()); + const queries: string[] = []; + const unsafe = db.$client.unsafe; + db.$client.unsafe = ((query: string, ...rest: unknown[]) => { + queries.push(query); + return (unsafe as (...args: unknown[]) => unknown).call(db.$client, query, ...rest); + }) as typeof unsafe; + onTestFinished(() => { db.$client.unsafe = unsafe; }); + const register = () => { + const tasks: Array<{ run(signal: AbortSignal): Promise; onSettled?(hints?: ReadonlySet): void }> = []; + f.checks.registerCheckWork({ register: (_queue: unknown, task: typeof tasks[number]) => { + tasks.push(task); return { wake() {}, stop: async () => {} }; + } } as never, () => true); + return tasks[0]; + }; + let task = register(); + const pulls = () => f.requests.filter(url => url.endsWith("/pulls/418")).length; + const pass = async (hint?: string) => { + clock += 1000; + if (hint) task.onSettled?.(new Set([hint])); + await task.run(new AbortController().signal); + }; + const window = async (seconds: number) => { + const before = { statements: queries.length, pulls: pulls(), + states: queries.filter(query => query.includes('from "chat_github_reviews"')).length }; + for (let second = 0; second < seconds; second++) await pass(); + return { statements: queries.length - before.statements, pulls: pulls() - before.pulls, + states: queries.filter(query => query.includes('from "chat_github_reviews"')).length - before.states }; + }; + return { ...f, pass, window, pulls, restart: () => { task = register(); }, now: () => clock }; + } + + it("backs off remote head polls while an unresolved GitHub review check cannot change", async () => { + const f = await unresolvedReviewCheckPasses(); + await f.pass(); + expect(f.writes).toEqual([expect.objectContaining({ status: "queued" })]); + const idle = await f.window(600); + expect(idle.pulls).toBeLessThanOrEqual(7); + expect(idle.states).toBeLessThanOrEqual(21); + expect(f.writes).toHaveLength(1); + const saved = await f.action(); + expect(saved).toMatchObject({ status: "received", result: { state: "queued", unchangedPolls: 6 } }); + expect(Date.parse(String(saved.result?.retryAt)) - f.now()).toBeLessThanOrEqual(GITHUB_REVIEW_CHECK_MAX_HEAD_POLL_MS); + }, 120_000); + + it("publishes a local change at once and still recovers a missed head after backing off", async () => { + const f = await unresolvedReviewCheckPasses(); + await f.window(400); + const backedOff = await f.action(); + expect(backedOff.result?.unchangedPolls).toBeGreaterThanOrEqual(5); + const pulls = f.pulls(); + f.restart(); + await f.pass("*"); + expect(f.pulls()).toBe(pulls); + await db.update(chatGitHubReviews).set({ runId: f.run.id }).where(eq(chatGitHubReviews.id, f.review.id)); + await f.pass(JSON.stringify({ companyId: f.companyId })); + expect(f.writes.at(-1)).toMatchObject({ status: "in_progress" }); + expect(await f.action()).toMatchObject({ status: "received", result: { state: "running", unchangedPolls: 0 } }); + await f.window(400); + f.setRemoteHead("c".repeat(40)); + await f.window(GITHUB_REVIEW_CHECK_MAX_HEAD_POLL_MS / 1000 + 1); + expect((await f.action()).status).toBe("cancelled"); + expect((await db.select().from(chatGitHubReviews).where(eq(chatGitHubReviews.id, f.review.id)))[0].state).toBe("superseded"); + }, 120_000); + async function reviewProjectionFixture(cancelFirst = true) { const f = await reviewBotFixture(); const deliveryId = randomUUID(); diff --git a/server/src/services/chat-github-checks.ts b/server/src/services/chat-github-checks.ts index 63e1f9321f8..be799375918 100644 --- a/server/src/services/chat-github-checks.ts +++ b/server/src/services/chat-github-checks.ts @@ -21,6 +21,10 @@ import { githubBotRequest, } from "./chat-github-client.js"; +export const GITHUB_REVIEW_CHECK_HEAD_POLL_MS = 5_000; +export const GITHUB_REVIEW_CHECK_MAX_HEAD_POLL_MS = 300_000; +const UNCHANGED_HEAD_POLL_LIMIT = Math.ceil(Math.log2(GITHUB_REVIEW_CHECK_MAX_HEAD_POLL_MS / GITHUB_REVIEW_CHECK_HEAD_POLL_MS)); + /** Check status follows the ordinary chat delivery/task/run. This outbox never * schedules an agent. Assessments are published by the governed review tool. */ export function githubReviewCheckService(db: Db, fetchImpl = fetch) { @@ -248,6 +252,10 @@ export function githubReviewCheckService(db: Db, fetchImpl = fetch) { ); }); } + const savedPolls = Number(fresh.result?.unchangedPolls); + const unchangedPolls = fresh.status === "received" && fresh.result?.state === state + ? Math.min(UNCHANGED_HEAD_POLL_LIMIT, (Number.isSafeInteger(savedPolls) && savedPolls > 0 ? savedPolls : 0) + 1) + : 0; await lease.commit(async (tx) => { await tx .update(chatActions) @@ -257,7 +265,8 @@ export function githubReviewCheckService(db: Db, fetchImpl = fetch) { state, attempts: 0, retryable: true, - retryAt: new Date(Date.now() + 5000).toISOString(), + unchangedPolls, + retryAt: new Date(Date.now() + Math.min(GITHUB_REVIEW_CHECK_MAX_HEAD_POLL_MS, GITHUB_REVIEW_CHECK_HEAD_POLL_MS * 2 ** unchangedPolls)).toISOString(), }, updatedAt: new Date(), })