import { randomUUID } from "node:crypto"; import { and, desc, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { costEvents, activityLog, agentRuntimeState, agentWakeupRequests, agents, companies, companySkills, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issueComments, issues, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; import { heartbeatService } from "../services/heartbeat.ts"; import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js"; import { runningProcesses } from "../adapters/index.ts"; const mockAdapterExecute = vi.hoisted(() => vi.fn(async () => ({ exitCode: 0, signal: null, timedOut: false, errorMessage: null, summary: "Issue rewake throttle test run.", provider: "test", model: "test-model", })), ); vi.mock("../adapters/index.ts", async () => { const actual = await vi.importActual("../adapters/index.ts"); return { ...actual, getServerAdapter: vi.fn(() => ({ supportsLocalAgentJwt: false, execute: mockAdapterExecute, })), }; }); const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( `Skipping embedded Postgres issue rewake throttle tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, ); } describeEmbeddedPostgres("heartbeat issue rewake throttle", () => { let db!: ReturnType; let heartbeat!: ReturnType; let tempDb: Awaited> | null = null; beforeAll(async () => { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-issue-rewake-throttle-"); db = createDb(tempDb.connectionString); heartbeat = heartbeatService(db); }, 20_000); afterEach(async () => { runningProcesses.clear(); // Await every in-flight background heartbeat run to quiescence before the // deletes below. A wakeup claims a run and dispatches its execution // fire-and-forget, and that run can dispatch a follow-up wakeup, so a run or // wakeup can still write heartbeat_runs and issues rows when teardown starts // and would race the deletes (a heartbeat_runs delete deadlocks on the ON // DELETE SET NULL cascade to issues). The shared drain also awaits an // in-flight wakeup that is still before run registration, which a plain run // table status poll cannot see. await drainHeartbeatRunsToQuiescence(db, heartbeat); // Post-run bookkeeping (run-event records, follow-up wake scheduling) can // still write for a moment after a run reaches a terminal status, so a // single delete sweep can hit a foreign-key violation when a late insert // lands between two deletes. Retry the sweep until it goes through clean. for (let attempt = 0; ; attempt += 1) { try { await db.delete(environmentLeases); await db.delete(issueComments); await db.delete(issues); await db.delete(heartbeatRunEvents); await db.delete(activityLog); await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agentWakeupRequests); await db.delete(agentRuntimeState); await db.delete(agents); await db.delete(environments); await db.delete(executionWorkspaces); await db.delete(companySkills); await db.delete(companies); break; } catch (error) { if (attempt >= 4) throw error; await new Promise((resolve) => setTimeout(resolve, 100)); } } }); afterAll(async () => { await tempDb?.cleanup(); }); async function seedCompanyAgentIssue() { const companyId = randomUUID(); const agentId = randomUUID(); const issueId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Paperclip", issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "responsible-user", }); await db.insert(agents).values({ id: agentId, companyId, name: "CodexCoder", role: "engineer", status: "active", adapterType: "codex_local", adapterConfig: {}, runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1, }, }, permissions: {}, }); await db.insert(issues).values({ id: issueId, companyId, title: "Interrupted import mission", status: "in_progress", priority: "medium", assigneeAgentId: agentId, responsibleUserId: "responsible-user", }); return { companyId, agentId, issueId }; } async function seedTerminalRun(input: { companyId: string; agentId: string; issueId: string; status?: string; finishedSecondsAgo: number; startedSecondsAgo?: number; sessionIdAfter?: string; }) { const runId = randomUUID(); const finishedAt = new Date(Date.now() - input.finishedSecondsAgo * 1000); const startedAt = input.startedSecondsAgo === undefined ? new Date(finishedAt.getTime() - 5_000) : new Date(Date.now() - input.startedSecondsAgo * 1000); await db.insert(heartbeatRuns).values({ id: runId, companyId: input.companyId, agentId: input.agentId, invocationSource: "assignment", status: input.status ?? "succeeded", responsibleUserId: "responsible-user", createdAt: startedAt, startedAt, finishedAt, sessionIdAfter: input.sessionIdAfter, contextSnapshot: { issueId: input.issueId, wakeReason: "issue_assigned" }, }); return runId; } function assignmentWake(agentId: string, issueId: string) { return heartbeat.wakeup(agentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned", payload: { issueId }, contextSnapshot: { issueId, wakeReason: "issue_assigned" }, requestedByActorType: "system", requestedByActorId: "test", }); } async function latestWakeRequest(agentId: string) { return db .select({ status: agentWakeupRequests.status, reason: agentWakeupRequests.reason, payload: agentWakeupRequests.payload, }) .from(agentWakeupRequests) .where(eq(agentWakeupRequests.agentId, agentId)) .orderBy(desc(agentWakeupRequests.requestedAt)) .limit(1) .then((rows) => rows[0] ?? null); } it("skips event-free re-wakes after consecutive no-progress runs and admits them again on new input", async () => { const { companyId, agentId, issueId } = await seedCompanyAgentIssue(); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 40 }); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 10 }); const throttledWake = await assignmentWake(agentId, issueId); expect(throttledWake).toBeNull(); const skipped = await latestWakeRequest(agentId); expect(skipped?.status).toBe("skipped"); expect(skipped?.reason).toBe("issue_rewake_throttled"); const heartbeatSkip = (skipped?.payload as Record | null)?.heartbeatSkip as | Record | undefined; expect(heartbeatSkip?.noProgressStreak).toBe(2); expect(typeof heartbeatSkip?.nextAllowedAt).toBe("string"); const runCount = await db .select({ count: sql`count(*)::int` }) .from(heartbeatRuns) .where(eq(heartbeatRuns.companyId, companyId)) .then((rows) => rows[0]?.count ?? 0); expect(runCount).toBe(2); // A board comment on the issue is new input: the next event-free wake is // admitted even though the streak has not been broken by a run. await db.insert(activityLog).values({ companyId, actorType: "user", actorId: "board-user", action: "issue.comment_added", entityType: "issue", entityId: issueId, }); const admittedWake = await assignmentWake(agentId, issueId); expect(admittedWake).not.toBeNull(); }); it("does not throttle system comment-driven wakes even during a no-progress streak", async () => { const { companyId, agentId, issueId } = await seedCompanyAgentIssue(); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 40 }); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 10 }); const commentWake = await heartbeat.wakeup(agentId, { source: "automation", triggerDetail: "system", reason: "issue_commented", payload: { issueId, commentId: randomUUID() }, contextSnapshot: { issueId, wakeReason: "issue_commented" }, requestedByActorType: "system", requestedByActorId: "test", }); expect(commentWake).not.toBeNull(); }); it("keeps agent comments throttled without hiding genuinely new human input", async () => { const { companyId, agentId, issueId } = await seedCompanyAgentIssue(); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 40 }); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 10 }); const agentCommentId = randomUUID(); await db.insert(activityLog).values({ companyId, actorType: "agent", actorId: randomUUID(), action: "issue.comment_added", entityType: "issue", entityId: issueId, }); const throttledAgentCommentWake = await heartbeat.wakeup(agentId, { source: "automation", triggerDetail: "system", reason: "issue_commented", payload: { issueId, commentId: agentCommentId }, contextSnapshot: { issueId, wakeReason: "issue_commented", wakeCommentId: agentCommentId, }, requestedByActorType: "agent", requestedByActorId: randomUUID(), }); expect(throttledAgentCommentWake).toBeNull(); expect((await latestWakeRequest(agentId))?.reason).toBe("issue_rewake_throttled"); await db.insert(activityLog).values({ companyId, actorType: "user", actorId: "board-user", action: "issue.comment_added", entityType: "issue", entityId: issueId, }); const nextAgentCommentId = randomUUID(); const admittedAfterHumanInput = await heartbeat.wakeup(agentId, { source: "automation", triggerDetail: "system", reason: "issue_commented", payload: { issueId, commentId: nextAgentCommentId }, contextSnapshot: { issueId, wakeReason: "issue_commented", wakeCommentId: nextAgentCommentId, }, requestedByActorType: "agent", requestedByActorId: randomUUID(), }); expect(admittedAfterHumanInput).not.toBeNull(); }); it("keeps agent-authored explicit resume comments inside the no-progress cooldown", async () => { const { companyId, agentId, issueId } = await seedCompanyAgentIssue(); const resumeFromRunId = await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 40, sessionIdAfter: randomUUID(), }); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 10 }); const commentId = randomUUID(); const resumeWake = await heartbeat.wakeup(agentId, { source: "automation", triggerDetail: "system", reason: "issue_reopened_via_comment", payload: { issueId, commentId, resumeFromRunId, resumeIntent: true }, contextSnapshot: { issueId, wakeReason: "issue_reopened_via_comment", wakeCommentId: commentId, resumeIntent: true, }, requestedByActorType: "agent", requestedByActorId: randomUUID(), }); expect(resumeWake).toBeNull(); expect((await latestWakeRequest(agentId))?.reason).toBe("issue_rewake_throttled"); }); it("does not throttle the wake that follows a failed run", async () => { const { companyId, agentId, issueId } = await seedCompanyAgentIssue(); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 70 }); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 40 }); await seedTerminalRun({ companyId, agentId, issueId, status: "failed", finishedSecondsAgo: 10 }); const recoveryWake = await assignmentWake(agentId, issueId); expect(recoveryWake).not.toBeNull(); }); it("does not throttle when a recent run produced issue-visible progress", async () => { const { companyId, agentId, issueId } = await seedCompanyAgentIssue(); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 40 }); const progressRunId = await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 10 }); await db.insert(activityLog).values({ companyId, actorType: "agent", actorId: agentId, agentId, runId: progressRunId, action: "issue.comment_added", entityType: "issue", entityId: issueId, createdAt: new Date(Date.now() - 11_000), }); const wake = await assignmentWake(agentId, issueId); expect(wake).not.toBeNull(); }); it("does not count progress on another issue toward the current issue", async () => { const { companyId, agentId, issueId } = await seedCompanyAgentIssue(); const otherIssueId = randomUUID(); await db.insert(issues).values({ id: otherIssueId, companyId, title: "Related follow-up", status: "in_progress", priority: "medium", assigneeAgentId: agentId, responsibleUserId: "responsible-user", }); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 40 }); const progressRunId = await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 10 }); await db.insert(activityLog).values({ companyId, actorType: "agent", actorId: agentId, agentId, runId: progressRunId, action: "issue.comment_added", entityType: "issue", entityId: otherIssueId, createdAt: new Date(Date.now() - 11_000), }); const wake = await assignmentWake(agentId, issueId); expect(wake).toBeNull(); expect((await latestWakeRequest(agentId))?.reason).toBe("issue_rewake_throttled"); }); it("counts a long-running session that finished inside the lookback window", async () => { const { companyId, agentId, issueId } = await seedCompanyAgentIssue(); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 40, startedSecondsAgo: 7 * 60 * 60, }); await seedTerminalRun({ companyId, agentId, issueId, finishedSecondsAgo: 10 }); const wake = await assignmentWake(agentId, issueId); expect(wake).toBeNull(); expect((await latestWakeRequest(agentId))?.reason).toBe("issue_rewake_throttled"); }); });