diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 15eb1d3917d..38e57bab9cb 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -244,10 +244,12 @@ Suspicious silence creates a medium-priority review issue for the selected recov Watchdog decisions are explicit operator/recovery-owner decisions: -- `snooze` records a future quiet-until time and suppresses scan-created review work during that window -- `continue` records that the current evidence is acceptable but does not cancel or mutate the active run +- `snooze` records an operator-chosen future quiet-until time and suppresses scan-created review work during that window +- `continue` records that the current evidence is acceptable, does not cancel or mutate the active run, and sets a 30-minute default re-arm window before the watchdog evaluates the still-silent run again - `dismissed_false_positive` records why the review was not actionable +Operators should prefer `snooze` for known time-bounded quiet periods. `continue` is only a short acknowledgement of the current evidence; if the run remains silent after the re-arm window, the periodic watchdog scan can create or update review work again. + The board can record watchdog decisions. The assigned owner of the watchdog evaluation issue can also record them. Other agents cannot. ## 11. Auto-Recover vs Explicit Recovery vs Human Escalation diff --git a/server/src/__tests__/heartbeat-active-run-output-watchdog.test.ts b/server/src/__tests__/heartbeat-active-run-output-watchdog.test.ts index 51038ef0224..45ba2c5d655 100644 --- a/server/src/__tests__/heartbeat-active-run-output-watchdog.test.ts +++ b/server/src/__tests__/heartbeat-active-run-output-watchdog.test.ts @@ -15,6 +15,7 @@ import { startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; import { + ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS, ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS, ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS, heartbeatService, @@ -352,6 +353,58 @@ describeEmbeddedPostgres("active-run output watchdog", () => { }); }); + it("re-arms continue decisions after the default quiet window", async () => { + const now = new Date("2026-04-22T20:00:00.000Z"); + const { companyId, managerId, runId } = await seedRunningRun({ + now, + ageMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS + 60_000, + }); + const heartbeat = heartbeatService(db); + const recovery = recoveryService(db, { enqueueWakeup: vi.fn() }); + + const scan = await heartbeat.scanSilentActiveRuns({ now, companyId }); + const evaluationIssueId = scan.evaluationIssueIds[0]; + expect(evaluationIssueId).toBeTruthy(); + + const decision = await recovery.recordWatchdogDecision({ + runId, + actor: { type: "agent", agentId: managerId }, + decision: "continue", + evaluationIssueId, + reason: "Current evidence is acceptable; keep watching.", + now, + }); + const rearmAt = new Date(now.getTime() + ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS); + expect(decision).toMatchObject({ + runId, + evaluationIssueId, + decision: "continue", + createdByAgentId: managerId, + }); + expect(decision.snoozedUntil?.toISOString()).toBe(rearmAt.toISOString()); + + await db.update(issues).set({ status: "done" }).where(eq(issues.id, evaluationIssueId)); + + const beforeRearm = await heartbeat.scanSilentActiveRuns({ + now: new Date(rearmAt.getTime() - 60_000), + companyId, + }); + expect(beforeRearm).toMatchObject({ created: 0, snoozed: 1 }); + + const afterRearm = await heartbeat.scanSilentActiveRuns({ + now: new Date(rearmAt.getTime() + 60_000), + companyId, + }); + expect(afterRearm.created).toBe(1); + expect(afterRearm.evaluationIssueIds[0]).not.toBe(evaluationIssueId); + + const evaluations = await db + .select() + .from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stale_active_run_evaluation"))); + expect(evaluations.filter((issue) => !["done", "cancelled"].includes(issue.status))).toHaveLength(1); + }); + it("rejects agent watchdog decisions using issues not bound to the target run", async () => { const now = new Date("2026-04-22T20:00:00.000Z"); const { companyId, managerId, coderId, runId, issuePrefix } = await seedRunningRun({ diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 5e6466891fa..e9fe107ab1f 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -155,6 +155,7 @@ const CANCELLABLE_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retr const HEARTBEAT_RUN_TERMINAL_STATUSES = ["succeeded", "failed", "cancelled", "timed_out"] as const; const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "timed_out"] as const; export { + ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS, ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS, ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS, } from "./recovery/service.js"; diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 5e9537bb7bc..87b687e9397 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -36,6 +36,7 @@ const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "ti const ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_MIN_STALE_MS = 24 * 60 * 60 * 1000; export const ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS = 60 * 60 * 1000; export const ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS = 4 * 60 * 60 * 1000; +export const ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS = 30 * 60 * 1000; const ACTIVE_RUN_OUTPUT_EVIDENCE_TAIL_BYTES = 8 * 1024; const STRANDED_ISSUE_RECOVERY_ORIGIN_KIND = RECOVERY_ORIGIN_KINDS.strandedIssueRecovery; const STALE_ACTIVE_RUN_EVALUATION_ORIGIN_KIND = RECOVERY_ORIGIN_KINDS.staleActiveRunEvaluation; @@ -509,7 +510,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) return startedAt ? Math.max(0, now.getTime() - startedAt.getTime()) : null; } - async function latestActiveOutputSnooze(companyId: string, runId: string, now = new Date()) { + async function latestActiveOutputQuietUntilDecision(companyId: string, runId: string, now = new Date()) { const [row] = await db .select() .from(heartbeatRunWatchdogDecisions) @@ -517,7 +518,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) and( eq(heartbeatRunWatchdogDecisions.companyId, companyId), eq(heartbeatRunWatchdogDecisions.runId, runId), - eq(heartbeatRunWatchdogDecisions.decision, "snooze"), + inArray(heartbeatRunWatchdogDecisions.decision, ["snooze", "continue"]), gt(heartbeatRunWatchdogDecisions.snoozedUntil, now), ), ) @@ -557,15 +558,15 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) >, now = new Date(), ): Promise { - const [snooze, evaluation] = await Promise.all([ - latestActiveOutputSnooze(run.companyId, run.id, now), + const [quietUntilDecision, evaluation] = await Promise.all([ + latestActiveOutputQuietUntilDecision(run.companyId, run.id, now), findOpenStaleRunEvaluation(run.companyId, run.id), ]); const silenceStartedAt = silenceStartedAtForRun(run); const silenceAgeMs = run.status === "running" ? silenceAgeMsForRun(run, now) : null; const level = run.status !== "running" ? "not_applicable" - : snooze + : quietUntilDecision ? "snoozed" : (silenceAgeMs ?? 0) >= ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS ? "critical" @@ -583,7 +584,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) level, suspicionThresholdMs: ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS, criticalThresholdMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS, - snoozedUntil: snooze?.snoozedUntil ?? null, + snoozedUntil: quietUntilDecision?.snoozedUntil ?? null, evaluationIssueId: evaluation?.id ?? null, evaluationIssueIdentifier: evaluation?.identifier ?? null, }; @@ -993,7 +994,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) }; for (const run of candidates) { - if (await latestActiveOutputSnooze(run.companyId, run.id, now)) { + if (await latestActiveOutputQuietUntilDecision(run.companyId, run.id, now)) { result.snoozed += 1; continue; } @@ -1018,6 +1019,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) reason?: string | null; snoozedUntil?: Date | null; createdByRunId?: string | null; + now?: Date; }) { const [run] = await db .select() @@ -1095,6 +1097,15 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) } } + const decisionNow = input.now ?? new Date(); + const effectiveSnoozedUntil = input.decision === "snooze" + ? input.snoozedUntil ?? null + : input.decision === "continue" + ? input.snoozedUntil && input.snoozedUntil > decisionNow + ? input.snoozedUntil + : new Date(decisionNow.getTime() + ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS) + : null; + const [row] = await db .insert(heartbeatRunWatchdogDecisions) .values({ @@ -1102,7 +1113,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) runId: run.id, evaluationIssueId: input.evaluationIssueId ?? null, decision: input.decision, - snoozedUntil: input.snoozedUntil ?? null, + snoozedUntil: effectiveSnoozedUntil, reason: input.reason ?? null, createdByAgentId: input.actor.type === "agent" ? input.actor.agentId ?? null : null, createdByUserId: input.actor.type === "board" ? input.actor.userId ?? null : null, @@ -1127,7 +1138,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) source: "recovery.record_watchdog_decision", decision: input.decision, evaluationIssueId: input.evaluationIssueId ?? null, - snoozedUntil: input.snoozedUntil?.toISOString() ?? null, + snoozedUntil: effectiveSnoozedUntil?.toISOString() ?? null, reason: input.reason ?? null, }, });