From 7560accda86cacc06ed8b8627c27a0f1eab9fcb2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 17:15:16 -0600 Subject: [PATCH] fix: report incomplete sleep schedule diagnostics Keep oversized catalogs blocked while distinguishing an incomplete inspection from known work. Preserve ordinary eligibility and SQL; verify both catalog bounds and mixed known and unknown reasons. Co-Authored-By: Paperclip --- .../src/__tests__/idle-scheduled-work.test.ts | 2 +- .../__tests__/idle-sleep-diagnostics.test.ts | 55 ++++++++++++++++++- server/src/services/idle-issue-schedule.ts | 3 +- server/src/services/idle-scheduled-work.ts | 3 +- server/src/services/idle-sleep-safety.ts | 8 +-- 5 files changed, 63 insertions(+), 8 deletions(-) diff --git a/server/src/__tests__/idle-scheduled-work.test.ts b/server/src/__tests__/idle-scheduled-work.test.ts index 11b2c9d6b25..bc91dd7b86c 100644 --- a/server/src/__tests__/idle-scheduled-work.test.ts +++ b/server/src/__tests__/idle-scheduled-work.test.ts @@ -24,7 +24,7 @@ describe("bounded idle routine catalog", () => { const formatter = vi.spyOn(Intl, "DateTimeFormat"); try { expect(await readIdleRoutineSchedule(database(Array(10_001).fill(trigger)), "owner", now)) - .toEqual({ blocked: true }); + .toEqual({ blocked: true, inspectionComplete: false }); expect(formatter).not.toHaveBeenCalled(); } finally { formatter.mockRestore(); } }); diff --git a/server/src/__tests__/idle-sleep-diagnostics.test.ts b/server/src/__tests__/idle-sleep-diagnostics.test.ts index 411caac4d47..58c6cc3bdba 100644 --- a/server/src/__tests__/idle-sleep-diagnostics.test.ts +++ b/server/src/__tests__/idle-sleep-diagnostics.test.ts @@ -13,11 +13,15 @@ const hold = (): IdleSleepDrainStatus => ({ ownerId, draining: true, startedAt: const localNone = async () => "none" as const; const collect = { collectDiagnostics: true } as const; -function database(present: string[] = [], change?: (row: Record) => unknown) { +function database(present: string[] = [], change?: (row: Record) => unknown, + schedules?: { routines?: unknown[]; issues?: unknown[] }) { let keys: string[] = []; const execute = vi.fn(async (statement: SQL) => { const query = new PgDialect().sqlToQuery(statement).sql; if (query.startsWith("SET LOCAL")) return []; + if (query.includes("FROM routine_triggers t JOIN routines r")) return schedules?.routines ?? []; + if (query.includes("FROM issues i JOIN companies c")) return schedules?.issues ?? []; + if (query.includes("WHERE status NOT IN ('active', 'paused', 'archived')")) return [{ blocked: false }]; if (query.endsWith(" AS blocked")) return [{ blocked: present.length > 0 }]; keys = [...query.matchAll(/ AS "([^"]+)"(?=, EXISTS |$)/g)].map(match => match[1]!); const row = Object.fromEntries(keys.map(key => [key, present.includes(key)])); @@ -60,6 +64,55 @@ describe("owned idle sleep diagnostics", () => { } }); + const routine = { id: "trigger", routine_id: "routine", company_id: "company", kind: "schedule", + setup_pending: false, cron_expression: "*/15 * * * *", timezone: "UTC", scoped: true, + next_run_at: new Date(now + 600_000), routine_updated: new Date(now), trigger_updated: new Date(now) }; + const issue = { id: "issue", company_id: "company", status: "backlog", updated_at: new Date(now), + assignee_agent_id: "agent", assignee_user_id: null, checkout_run_id: null, execution_run_id: null, + execution_policy: null, execution_state: null, unblock_descriptor: null, conversation_agent_id: null, + monitor_next_check_at: null, monitor_wake_requested_at: null, monitor_attempt_count: 0, + company_status: "active", company_updated: new Date(now), agent_status: "idle", agent_updated: new Date(now), + scoped: true, unsafe_dependencies: false, dependency_ready: false, untouched_onboarding: false, + response_driven_wait: false, unblock_request_finished: false, response_wait_generation: null }; + const scheduledRead = (f: ReturnType, diagnostics = true) => readIdleSleepSafety(f.db, hold, + () => now, ownerId, localNone, undefined, undefined, true, true, diagnostics ? collect : undefined); + + it.each(["routines", "issues"] as const)("marks an overflowing %s inspection unknown without relaxing ordinary eligibility", async family => { + const f = database([], undefined, { [family]: Array(10_001).fill(family === "routines" ? routine : issue) }); + expect(await scheduledRead(f, false)).toEqual({ version: 1, backgroundWork: "present" }); + const result = await scheduledRead(f); + expect(result.backgroundWork).toBe("unknown"); + expect(result.diagnostics?.complete).toBe(false); + expect(result.diagnostics?.checks).toContainEqual({ reason: `${family}.schedule`, state: "unknown" }); + }); + + it.each(["routines", "issues"] as const)("fully inspects a %s catalog at the 10,000-row bound", async family => { + const f = database([], undefined, { [family]: Array(10_000).fill(family === "routines" ? routine : issue) }); + const result = await scheduledRead(f); + expect(result.backgroundWork).toBe("none"); + expect(result.diagnostics?.complete).toBe(true); + expect(result.diagnostics?.checks).toContainEqual({ reason: `${family}.schedule`, state: "none" }); + }); + + it.each(["routines", "issues"] as const)("retains an actual %s blocker as known work", async family => { + const f = database([], undefined, { [family]: [family === "routines" ? { ...routine, next_run_at: new Date(now) } + : { ...issue, status: "todo" }] }); + const result = await scheduledRead(f); + expect(result.backgroundWork).toBe("present"); + expect(result.diagnostics?.complete).toBe(true); + expect(result.diagnostics?.checks).toContainEqual({ reason: `${family}.schedule`, state: "present" }); + }); + + it.each(["routines", "issues"] as const)("keeps known durable work present alongside an incomplete %s inspection", async family => { + const f = database(["heartbeat_runs"], undefined, { [family]: Array(10_001).fill(family === "routines" ? routine : issue) }); + const result = await scheduledRead(f); + expect(result.backgroundWork).toBe("present"); + expect(result.diagnostics?.complete).toBe(false); + expect(result.diagnostics?.checks).toEqual(expect.arrayContaining([ + { reason: "heartbeat_runs", state: "present" }, { reason: `${family}.schedule`, state: "unknown" }, + ])); + }); + it("preserves ordinary local and plugin short circuits and response shape", async () => { const f = database(); const plugins = vi.fn(async () => ({ backgroundWork: "present" as const, pluginIds: [] })); diff --git a/server/src/services/idle-issue-schedule.ts b/server/src/services/idle-issue-schedule.ts index c234403e873..5037c099d97 100644 --- a/server/src/services/idle-issue-schedule.ts +++ b/server/src/services/idle-issue-schedule.ts @@ -15,6 +15,7 @@ const record = (value: unknown): Record | null => export async function readIdleIssueSchedule(tx: Pick, ownerId: string, now: number): Promise<{ blocked: boolean; + inspectionComplete?: false; scheduledWake?: ScheduledWake; generation?: string; }> { @@ -53,7 +54,7 @@ export async function readIdleIssueSchedule(tx: Pick, ownerId: st FROM issues i JOIN companies c ON c.id = i.company_id LEFT JOIN agents a ON a.id = i.assignee_agent_id AND a.company_id = i.company_id WHERE i.status NOT IN ('done', 'cancelled') ORDER BY i.id LIMIT ${limit + 1}`); - if (rows.length > limit) return { blocked: true }; + if (rows.length > limit) return { blocked: true, inspectionComplete: false }; let earliest = Infinity; for (const row of rows) { const policy = record(row.execution_policy); diff --git a/server/src/services/idle-scheduled-work.ts b/server/src/services/idle-scheduled-work.ts index 6e9ed7f670b..f7f8c2b4b1d 100644 --- a/server/src/services/idle-scheduled-work.ts +++ b/server/src/services/idle-scheduled-work.ts @@ -15,6 +15,7 @@ export interface ScheduledWake { export async function readIdleRoutineSchedule(tx: Pick, ownerId: string, now: number): Promise<{ blocked: boolean; + inspectionComplete?: false; scheduledWake?: ScheduledWake; }> { const unknown = await tx.execute<{ blocked: boolean }>(sql`SELECT EXISTS ( @@ -33,7 +34,7 @@ export async function readIdleRoutineSchedule(tx: Pick, ownerId: FROM routine_triggers t JOIN routines r ON r.id = t.routine_id WHERE r.status = 'active' AND t.enabled AND NOT t.archived ORDER BY t.id LIMIT ${limit + 1}`); - if (rows.length > limit) return { blocked: true }; + if (rows.length > limit) return { blocked: true, inspectionComplete: false }; let earliest = Infinity; const validTimezones = new Set(); const validCrons = new Set(); diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index 0eafd3b3383..c773958a546 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -390,17 +390,17 @@ export async function readIdleSleepSafety( if (scheduledRoutines) { if (!owned()) return undefined; const schedule = await readIdleRoutineSchedule(tx, ownerId!, now()); - diagnostic?.record("routines.schedule", schedule.blocked ? "present" : "none"); + diagnostic?.record("routines.schedule", schedule.inspectionComplete === false ? "unknown" : schedule.blocked ? "present" : "none"); if (schedule.blocked && !diagnostic) return true; - found ||= schedule.blocked; + found ||= schedule.blocked && schedule.inspectionComplete !== false; scheduledWake = schedule.scheduledWake; } if (scheduledIssueMonitors) { if (!scheduledRoutines || !owned()) return undefined; const schedule = await readIdleIssueSchedule(tx, ownerId!, now()); - diagnostic?.record("issues.schedule", schedule.blocked ? "present" : "none"); + diagnostic?.record("issues.schedule", schedule.inspectionComplete === false ? "unknown" : schedule.blocked ? "present" : "none"); if (schedule.blocked && !diagnostic) return true; - found ||= schedule.blocked; + found ||= schedule.blocked && schedule.inspectionComplete !== false; scheduledWake = combineIdleSchedules(scheduledWake, schedule.scheduledWake, schedule.generation); } return found;