diff --git a/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts b/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts index 369bb05c2b7..58dfee3d9fd 100644 --- a/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts +++ b/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts @@ -117,7 +117,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => { }); } - async function seedBlockedChain(opts: { outsideLookback?: boolean } = {}) { + async function seedBlockedChain(opts: { outsideLookback?: boolean; freshUpdateOutsideLookback?: boolean } = {}) { const companyId = randomUUID(); const managerId = randomUUID(); const coderId = randomUUID(); @@ -161,6 +161,9 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => { const issueTimestamp = opts.outsideLookback === true ? new Date(Date.now() - 25 * 60 * 60 * 1000) : new Date(Date.now() - 60 * 60 * 1000); + const updatedAt = opts.freshUpdateOutsideLookback === true + ? new Date(Date.now() - 60 * 60 * 1000) + : issueTimestamp; await db.insert(issues).values([ { id: blockedIssueId, @@ -172,7 +175,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => { issueNumber: 1, identifier: `${issuePrefix}-1`, createdAt: issueTimestamp, - updatedAt: issueTimestamp, + updatedAt, }, { id: blockerIssueId, @@ -183,7 +186,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => { issueNumber: 2, identifier: `${issuePrefix}-2`, createdAt: issueTimestamp, - updatedAt: issueTimestamp, + updatedAt, }, ]); @@ -236,6 +239,27 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => { expect(escalations).toHaveLength(0); }); + it("does not create recovery issues for old legacy findings that were only updated recently", async () => { + await enableAutoRecovery(); + const { companyId } = await seedBlockedChain({ + outsideLookback: true, + freshUpdateOutsideLookback: true, + }); + const heartbeat = heartbeatService(db); + + const result = await heartbeat.reconcileIssueGraphLiveness(); + + expect(result.findings).toBe(1); + expect(result.escalationsCreated).toBe(0); + expect(result.skippedOutsideLookback).toBe(1); + + const escalations = await db + .select() + .from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation"))); + expect(escalations).toHaveLength(0); + }); + it("suppresses liveness escalation when the source issue is under an active pause hold", async () => { await enableAutoRecovery(); const { companyId, blockedIssueId } = await seedBlockedChain(); diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 8be5b31997d..d8ca5db63ac 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -2414,31 +2414,36 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) return `${companyId}:${issueId}`; } - async function loadLivenessDependencyUpdatedAtByIssue(findings: IssueLivenessFinding[]) { + type LivenessDependencyTimestamps = { + createdAt: Date; + updatedAt: Date; + }; + + async function loadLivenessDependencyTimestampsByIssue(findings: IssueLivenessFinding[]) { const issueIds = [ ...new Set( findings.flatMap((finding) => finding.dependencyPath.map((entry) => entry.issueId)), ), ]; - if (issueIds.length === 0) return new Map(); + if (issueIds.length === 0) return new Map(); const rows = await db - .select({ id: issues.id, companyId: issues.companyId, updatedAt: issues.updatedAt }) + .select({ id: issues.id, companyId: issues.companyId, createdAt: issues.createdAt, updatedAt: issues.updatedAt }) .from(issues) .where(inArray(issues.id, issueIds)); return new Map(rows.map((row) => [ livenessDependencyIssueKey(row.companyId, row.id), - row.updatedAt, + { createdAt: row.createdAt, updatedAt: row.updatedAt }, ])); } function latestDependencyUpdatedAtForLivenessFinding( finding: IssueLivenessFinding, - updatedAtByIssueKey: Map, + timestampsByIssueKey: Map, ) { const dependencyIssueIds = [...new Set(finding.dependencyPath.map((entry) => entry.issueId))]; if (dependencyIssueIds.length === 0) return null; const timestamps = dependencyIssueIds.map((issueId) => - updatedAtByIssueKey.get(livenessDependencyIssueKey(finding.companyId, issueId)) ?? null + timestampsByIssueKey.get(livenessDependencyIssueKey(finding.companyId, issueId))?.updatedAt ?? null ); if (timestamps.some((timestamp) => !timestamp)) return null; const [firstTimestamp, ...remainingTimestamps] = timestamps as Date[]; @@ -2447,13 +2452,22 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) firstTimestamp!); } + function recoveryIssueCreatedAtForLivenessFinding( + finding: IssueLivenessFinding, + timestampsByIssueKey: Map, + ) { + return timestampsByIssueKey.get( + livenessDependencyIssueKey(finding.companyId, finding.recoveryIssueId), + )?.createdAt ?? null; + } + function isLivenessFindingInsideAutoRecoveryLookback( finding: IssueLivenessFinding, cutoff: Date, - updatedAtByIssueKey: Map, + timestampsByIssueKey: Map, ) { - const latestUpdatedAt = latestDependencyUpdatedAtForLivenessFinding(finding, updatedAtByIssueKey); - return Boolean(latestUpdatedAt && latestUpdatedAt >= cutoff); + const recoveryIssueCreatedAt = recoveryIssueCreatedAtForLivenessFinding(finding, timestampsByIssueKey); + return Boolean(recoveryIssueCreatedAt && recoveryIssueCreatedAt >= cutoff); } async function buildIssueGraphLivenessAutoRecoveryPreview( @@ -2463,7 +2477,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) const lookbackHours = normalizeIssueGraphLivenessAutoRecoveryLookbackHours(opts?.lookbackHours); const cutoff = new Date(now.getTime() - lookbackHours * 60 * 60 * 1000); const findings = await collectIssueGraphLivenessFindings(); - const updatedAtByIssueKey = await loadLivenessDependencyUpdatedAtByIssue(findings); + const timestampsByIssueKey = await loadLivenessDependencyTimestampsByIssue(findings); const issueIds = [...new Set(findings.map((finding) => finding.recoveryIssueId))]; const recoveryRows = issueIds.length > 0 ? await db @@ -2478,9 +2492,12 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) for (const finding of findings) { const latestDependencyUpdatedAt = latestDependencyUpdatedAtForLivenessFinding( finding, - updatedAtByIssueKey, + timestampsByIssueKey, ); - if (!latestDependencyUpdatedAt || latestDependencyUpdatedAt < cutoff) { + if ( + !latestDependencyUpdatedAt || + !isLivenessFindingInsideAutoRecoveryLookback(finding, cutoff, timestampsByIssueKey) + ) { skippedOutsideLookback += 1; continue; } @@ -2790,7 +2807,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) const now = new Date(); const cutoff = new Date(now.getTime() - lookbackHours * 60 * 60 * 1000); const obsoleteRecoveryCleanup = await retireObsoleteLivenessRecoveryIssues(findings); - const updatedAtByIssueKey = await loadLivenessDependencyUpdatedAtByIssue(findings); + const timestampsByIssueKey = await loadLivenessDependencyTimestampsByIssue(findings); const result = { findings: findings.length, autoRecoveryEnabled, @@ -2815,7 +2832,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) } for (const finding of findings) { - if (!isLivenessFindingInsideAutoRecoveryLookback(finding, cutoff, updatedAtByIssueKey)) { + if (!isLivenessFindingInsideAutoRecoveryLookback(finding, cutoff, timestampsByIssueKey)) { result.skippedOutsideLookback += 1; result.skipped += 1; continue;