From 3f895b84032c2ec4cd9394474c7932bd10ba7261 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 19 Oct 2026 14:20:22 -0500 Subject: [PATCH] test(heartbeat): reproduce invisible monitor dispatch deferrals Co-Authored-By: Paperclip --- .../__tests__/issue-monitor-scheduler.test.ts | 34 +++++++++++++++++++ .../src/services/heartbeat/scheduling.test.ts | 26 +++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/issue-monitor-scheduler.test.ts b/server/src/__tests__/issue-monitor-scheduler.test.ts index 323fc7f7956..4cfe7d95801 100644 --- a/server/src/__tests__/issue-monitor-scheduler.test.ts +++ b/server/src/__tests__/issue-monitor-scheduler.test.ts @@ -311,6 +311,40 @@ describeEmbeddedPostgres("issue monitor scheduler", () => { expect((await db.select().from(agentWakeupRequests)).filter(wake => wake.reason === "issue_monitor_due")).toHaveLength(1); }); + it("explains held monitor dispatch without replaying uncertain execution", async () => { + const { companyId, issueId, agentId, nextCheckAt } = await seedFixture(); + const sourceRunId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId, agentId, + runtimeMode: "legacy", status: "interrupted", errorCode: "process_lost", + finishedAt: new Date(), contextSnapshot: { issueId } }); + const [hold] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, + kind: "active_run_watchdog", status: "active", ownerType: "board", + cause: "legacy_execution_requires_reconciliation", fingerprint: sourceRunId, + evidence: { runId: sourceRunId }, nextAction: "Inspect the interrupted execution before continuing." }).returning(); + const heartbeat = heartbeatService(db); + const tickAt = new Date("2026-04-11T12:31:00.000Z"); + expect((await heartbeat.tickTimers(tickAt)).enqueued).toBe(0); + expect((await heartbeat.tickTimers(new Date(tickAt.getTime() + 30_000))).enqueued).toBe(0); + const [waiting] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(waiting.monitorNextCheckAt).toEqual(nextCheckAt); + expect(waiting.monitorWakeRequestedAt).toBeNull(); + expect(waiting.monitorAttemptCount).toBe(0); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId))).toHaveLength(1); + const diagnostics = (await db.select().from(activityLog).where(eq(activityLog.entityId, issueId))) + .filter(row => row.action === "issue.monitor_deferred"); + expect(diagnostics).toHaveLength(1); + expect(diagnostics[0]?.details).toMatchObject({ nextCheckAt: nextCheckAt.toISOString(), + reason: "execution_reconciliation_required", blockerCause: "legacy_execution_requires_reconciliation", attemptCount: 0 }); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, hold!.id)))[0]?.status).toBe("active"); + await db.update(issueRecoveryActions).set({ status: "resolved", resolvedAt: new Date(), + evidence: { runId: sourceRunId, executionReconciliation: { action: "continue" } } }).where(eq(issueRecoveryActions.id, hold!.id)); + expect((await heartbeat.tickTimers(new Date(tickAt.getTime() + 60_000))).enqueued).toBe(1); + expect((await heartbeat.tickTimers(new Date(tickAt.getTime() + 90_000))).enqueued).toBe(0); + const [triggered] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(triggered.monitorAttemptCount).toBe(1); + expect(triggered.monitorNextCheckAt).toBeNull(); + }); + it("recovers a rejected native monitor call and delivers the corrected schedule once", async () => { const { companyId, issueId, agentId } = await seedFixture(); const runId = randomUUID(); diff --git a/server/src/services/heartbeat/scheduling.test.ts b/server/src/services/heartbeat/scheduling.test.ts index 8c1bb544da6..74d5a62bb40 100644 --- a/server/src/services/heartbeat/scheduling.test.ts +++ b/server/src/services/heartbeat/scheduling.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import type { Db } from "@paperclipai/db"; import { - agents, agentSessionGoalActions, agentTaskSessions, companies, createDb, + activityLog, agents, agentSessionGoalActions, agentTaskSessions, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, } from "@paperclipai/db"; import { eq, sql } from "drizzle-orm"; @@ -201,6 +201,30 @@ describe.skipIf(!postgres.supported)("heartbeat scheduling database boundary", ( expect(await f.scheduler.tickTimers(now)).toEqual({ checked: 1, enqueued: 1, skipped: 0 }); }); + it("records bounded dispatch failure diagnostics while retaining the schedule", async () => { + const f = await fixture(); + f.deps.enqueueWakeup.mockRejectedValue(new Error("PRIVATE_PROVIDER_FAILURE")); + await f.scheduler.tickTimers(now); + await f.scheduler.tickTimers(new Date(now.getTime() + 30_000)); + expect(await readIssue(f.issue.id)).toMatchObject({ monitorNextCheckAt: dueAt, monitorWakeRequestedAt: null, monitorAttemptCount: 0 }); + const events = await db.select().from(activityLog).where(eq(activityLog.entityId, f.issue.id)); + expect(events.filter(event => event.action === "issue.monitor_dispatch_failed")).toHaveLength(1); + expect(events[0]?.details).toMatchObject({ reason: "monitor_dispatch_failed", nextCheckAt: dueAt.toISOString(), attemptCount: 0 }); + expect(JSON.stringify(events)).not.toContain("PRIVATE_PROVIDER_FAILURE"); + }); + + it("does not attach old dispatch diagnostics to a replacement monitor", async () => { + const f = await fixture(); + const replacement = new Date(now.getTime() + 60_000); + f.deps.enqueueWakeup.mockImplementationOnce(async () => { + await db.update(issues).set({ monitorNextCheckAt: replacement, monitorWakeRequestedAt: null }).where(eq(issues.id, f.issue.id)); + return null; + }); + await f.scheduler.tickTimers(now); + expect(await readIssue(f.issue.id)).toMatchObject({ monitorNextCheckAt: replacement }); + expect(await db.select().from(activityLog).where(eq(activityLog.entityId, f.issue.id))).toHaveLength(0); + }); + it("leaves monitors alone while a native run owns the issue", async () => { const f = await fixture(); const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, nativeIssueId: f.issue.id, runtimeMode: "native", status: "running", contextSnapshot: { issueId: f.issue.id } }).returning();