diff --git a/packages/shared/src/types/dashboard.ts b/packages/shared/src/types/dashboard.ts index 977b124ce18..cdeaebdb777 100644 --- a/packages/shared/src/types/dashboard.ts +++ b/packages/shared/src/types/dashboard.ts @@ -7,10 +7,10 @@ export interface DashboardRunActivityDay { */ failed: number; /** - * Runs that terminated in a failure state (failed/timed_out) but whose retry - * chain eventually succeeded — e.g. restart-killed runs that recovered. Kept - * out of `failed` so the headline failure count reflects true, unrecovered - * failures. + * Runs that terminated as failed, timed_out, or interrupted (shutdown) but + * whose retry chain eventually succeeded — e.g. restart-killed runs and + * shutdown interrupts that recovered. Kept out of `failed` so the headline + * failure count reflects true, unrecovered failures. */ recovered: number; other: number; diff --git a/server/src/__tests__/dashboard-service.test.ts b/server/src/__tests__/dashboard-service.test.ts index 8642844ce5d..71b6ff9cd24 100644 --- a/server/src/__tests__/dashboard-service.test.ts +++ b/server/src/__tests__/dashboard-service.test.ts @@ -274,4 +274,54 @@ describeEmbeddedPostgres("dashboard service", () => { // process_lost kills that recovered must not leak into the failed breakdown. expect(bucket?.failedByErrorCode.process_lost).toBeUndefined(); }); + + it("counts a recovered shutdown interrupt toward recovered, not other", async () => { + const companyId = randomUUID(); + const agentId = randomUUID(); + const day = utcDay(-1); + const shutdown = randomUUID(); + const shutdownRetry = randomUUID(); + const stranded = randomUUID(); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "CodexCoder", + role: "engineer", + status: "running", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + + const base = { + companyId, + agentId, + invocationSource: "assignment" as const, + createdAt: day, + }; + await db.insert(heartbeatRuns).values([ + { ...base, id: shutdown, status: "interrupted", errorCode: "server_shutdown_interrupted" }, + { ...base, id: shutdownRetry, status: "succeeded", retryOfRunId: shutdown }, + { ...base, id: stranded, status: "interrupted", errorCode: "server_shutdown_interrupted" }, + ]); + + const summary = await dashboardService(db).summary(companyId); + const bucket = summary.runActivity.find((entry) => entry.date === utcDateKey(day)); + expect(bucket).toMatchObject({ + succeeded: 1, + recovered: 1, + failed: 0, + other: 1, + total: 3, + failedByErrorCode: {}, + }); + }); }); diff --git a/server/src/services/dashboard.ts b/server/src/services/dashboard.ts index 9da5d08aa4f..afd7ee25c58 100644 --- a/server/src/services/dashboard.ts +++ b/server/src/services/dashboard.ts @@ -103,9 +103,10 @@ export function dashboardService(db: Db) { const monthSpendCents = Number(monthSpend); // Per-day run breakdown. A run is "recovered" when its retry chain later // succeeded (recovered_runs = all ancestors of a succeeded retry), so a - // restart-killed run whose retry succeeded is pulled out of the headline - // failed count. error_code is carried through so a failure spike can be - // attributed to an error class (e.g. process_lost, provider_quota). + // restart-killed or shutdown-interrupted run whose retry succeeded is + // pulled out of the headline failure count. error_code is carried through + // so a failure spike can be attributed to an error class (e.g. process_lost, + // provider_quota). // Both recursive arms are bounded to the chart window: a retry is always // created after the run it retries, so ancestors of an out-of-window // child are themselves out of window and invisible to the membership @@ -166,7 +167,11 @@ export function dashboardService(db: Db) { const recovered = row.recovered === true || row.recovered === "t" || row.recovered === "true"; if (status === "succeeded") { bucket.succeeded += count; - } else if (status === "failed" || status === "timed_out") { + } else if ( + status === "failed" || + status === "timed_out" || + (recovered && status === "interrupted") + ) { if (recovered) { bucket.recovered += count; } else {