From 2f9384cc0deb79fad3c6c375eee625b88a30112e Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 27 Apr 2026 08:26:02 -0500 Subject: [PATCH] Fix liveness auto-recovery review feedback Co-Authored-By: Paperclip --- doc/execution-semantics.md | 2 + packages/shared/src/validators/instance.ts | 2 +- server/src/services/instance-settings.ts | 4 +- server/src/services/recovery/service.ts | 60 ++++++++++++++++------ 4 files changed, 50 insertions(+), 18 deletions(-) diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 869d4bc8f25..b014253b85b 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -326,6 +326,8 @@ Examples: The source issue remains visible and blocked on the recovery issue when blocking is necessary for correctness. The recovery owner must restore a live path, resolve the source issue manually, or record the reason it is a false positive. +Instance-level issue-graph liveness auto-recovery is disabled by default. When enabled, its lookback window means "dependency paths updated within the last N hours"; older findings remain advisory and are counted as outside the configured lookback instead of creating recovery issues automatically. This is an operator noise control, not the older staleness delay for determining whether a chain is old enough to surface. + ### Human Escalation Human escalation is required when the next safe action depends on board judgment, budget/approval policy, or information unavailable to the control plane. diff --git a/packages/shared/src/validators/instance.ts b/packages/shared/src/validators/instance.ts index ffa947b5dc0..3415539a2ad 100644 --- a/packages/shared/src/validators/instance.ts +++ b/packages/shared/src/validators/instance.ts @@ -39,7 +39,7 @@ export const instanceExperimentalSettingsSchema = z.object({ enableEnvironments: z.boolean().default(false), enableIsolatedWorkspaces: z.boolean().default(false), autoRestartDevServerWhenIdle: z.boolean().default(false), - enableIssueGraphLivenessAutoRecovery: z.boolean().default(true), + enableIssueGraphLivenessAutoRecovery: z.boolean().default(false), issueGraphLivenessAutoRecoveryLookbackHours: z .number() .int() diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 455eeec6d6d..c447a920efd 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -42,7 +42,7 @@ function normalizeExperimentalSettings(raw: unknown): InstanceExperimentalSettin enableEnvironments: parsed.data.enableEnvironments ?? false, enableIsolatedWorkspaces: parsed.data.enableIsolatedWorkspaces ?? false, autoRestartDevServerWhenIdle: parsed.data.autoRestartDevServerWhenIdle ?? false, - enableIssueGraphLivenessAutoRecovery: parsed.data.enableIssueGraphLivenessAutoRecovery ?? true, + enableIssueGraphLivenessAutoRecovery: parsed.data.enableIssueGraphLivenessAutoRecovery ?? false, issueGraphLivenessAutoRecoveryLookbackHours: parsed.data.issueGraphLivenessAutoRecoveryLookbackHours ?? DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS, @@ -52,7 +52,7 @@ function normalizeExperimentalSettings(raw: unknown): InstanceExperimentalSettin enableEnvironments: false, enableIsolatedWorkspaces: false, autoRestartDevServerWhenIdle: false, - enableIssueGraphLivenessAutoRecovery: true, + enableIssueGraphLivenessAutoRecovery: false, issueGraphLivenessAutoRecoveryLookbackHours: DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS, }; diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index ce308ef7d38..f6e8c1f7d69 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -1871,24 +1871,49 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) ); } - async function latestDependencyUpdatedAtForLivenessFinding(finding: IssueLivenessFinding) { - const issueIds = [...new Set(finding.dependencyPath.map((entry) => entry.issueId))]; - if (issueIds.length === 0) return null; - const rows = await db - .select({ id: issues.id, updatedAt: issues.updatedAt }) - .from(issues) - .where(and(eq(issues.companyId, finding.companyId), inArray(issues.id, issueIds))); - if (rows.length !== issueIds.length) return null; - return rows.reduce((latest, row) => - row.updatedAt > latest ? row.updatedAt : latest, - rows[0]!.updatedAt); + function livenessDependencyIssueKey(companyId: string, issueId: string) { + return `${companyId}:${issueId}`; } - async function isLivenessFindingInsideAutoRecoveryLookback( + async function loadLivenessDependencyUpdatedAtByIssue(findings: IssueLivenessFinding[]) { + const issueIds = [ + ...new Set( + findings.flatMap((finding) => finding.dependencyPath.map((entry) => entry.issueId)), + ), + ]; + if (issueIds.length === 0) return new Map(); + const rows = await db + .select({ id: issues.id, companyId: issues.companyId, updatedAt: issues.updatedAt }) + .from(issues) + .where(inArray(issues.id, issueIds)); + return new Map(rows.map((row) => [ + livenessDependencyIssueKey(row.companyId, row.id), + row.updatedAt, + ])); + } + + function latestDependencyUpdatedAtForLivenessFinding( + finding: IssueLivenessFinding, + updatedAtByIssueKey: Map, + ) { + const dependencyIssueIds = [...new Set(finding.dependencyPath.map((entry) => entry.issueId))]; + if (dependencyIssueIds.length === 0) return null; + const timestamps = dependencyIssueIds.map((issueId) => + updatedAtByIssueKey.get(livenessDependencyIssueKey(finding.companyId, issueId)) ?? null + ); + if (timestamps.some((timestamp) => !timestamp)) return null; + const [firstTimestamp, ...remainingTimestamps] = timestamps as Date[]; + return remainingTimestamps.reduce((latest, updatedAt) => + updatedAt > latest ? updatedAt : latest, + firstTimestamp!); + } + + function isLivenessFindingInsideAutoRecoveryLookback( finding: IssueLivenessFinding, cutoff: Date, + updatedAtByIssueKey: Map, ) { - const latestUpdatedAt = await latestDependencyUpdatedAtForLivenessFinding(finding); + const latestUpdatedAt = latestDependencyUpdatedAtForLivenessFinding(finding, updatedAtByIssueKey); return Boolean(latestUpdatedAt && latestUpdatedAt >= cutoff); } @@ -1899,6 +1924,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) const lookbackHours = normalizeIssueGraphLivenessAutoRecoveryLookbackHours(opts?.lookbackHours); const cutoff = new Date(now.getTime() - lookbackHours * 60 * 60 * 1000); const findings = await collectIssueGraphLivenessFindings(); + const updatedAtByIssueKey = await loadLivenessDependencyUpdatedAtByIssue(findings); const issueIds = [...new Set(findings.map((finding) => finding.recoveryIssueId))]; const recoveryRows = issueIds.length > 0 ? await db @@ -1911,7 +1937,10 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) let skippedOutsideLookback = 0; for (const finding of findings) { - const latestDependencyUpdatedAt = await latestDependencyUpdatedAtForLivenessFinding(finding); + const latestDependencyUpdatedAt = latestDependencyUpdatedAtForLivenessFinding( + finding, + updatedAtByIssueKey, + ); if (!latestDependencyUpdatedAt || latestDependencyUpdatedAt < cutoff) { skippedOutsideLookback += 1; continue; @@ -2222,6 +2251,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) const now = new Date(); const cutoff = new Date(now.getTime() - lookbackHours * 60 * 60 * 1000); const obsoleteRecoveryCleanup = await retireObsoleteLivenessRecoveryIssues(findings); + const updatedAtByIssueKey = await loadLivenessDependencyUpdatedAtByIssue(findings); const result = { findings: findings.length, autoRecoveryEnabled, @@ -2246,7 +2276,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) } for (const finding of findings) { - if (!await isLivenessFindingInsideAutoRecoveryLookback(finding, cutoff)) { + if (!isLivenessFindingInsideAutoRecoveryLookback(finding, cutoff, updatedAtByIssueKey)) { result.skippedOutsideLookback += 1; result.skipped += 1; continue;