mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 05:11:49 +01:00
Add configurable liveness auto-recovery controls
Co-Authored-By: Paperclip <noreply@paperclip.ing> (cherry picked from commit 3f1618b5f1b63f337fa1739c2bfe37bb2ac89835)
This commit is contained in:
Dotta committed
1 parent
7f2f0bc64b
commit
5bc30ddfdd
13 files changed
+645
-52
No files matched your search
@@ -264,6 +264,8 @@ export type {
|
||||
InstanceExperimentalSettings,
|
||||
InstanceGeneralSettings,
|
||||
InstanceSettings,
|
||||
IssueGraphLivenessAutoRecoveryPreview,
|
||||
IssueGraphLivenessAutoRecoveryPreviewItem,
|
||||
BackupRetentionPolicy,
|
||||
Agent,
|
||||
AgentAccessState,
|
||||
@@ -548,6 +550,9 @@ export {
|
||||
WEEKLY_RETENTION_PRESETS,
|
||||
MONTHLY_RETENTION_PRESETS,
|
||||
DEFAULT_BACKUP_RETENTION,
|
||||
DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
MIN_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
MAX_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
} from "./types/instance.js";
|
||||
|
||||
export {
|
||||
@@ -561,7 +566,9 @@ export {
|
||||
type PatchInstanceGeneralSettings,
|
||||
instanceExperimentalSettingsSchema,
|
||||
patchInstanceExperimentalSettingsSchema,
|
||||
issueGraphLivenessAutoRecoveryRequestSchema,
|
||||
type PatchInstanceExperimentalSettings,
|
||||
type IssueGraphLivenessAutoRecoveryRequest,
|
||||
} from "./validators/index.js";
|
||||
|
||||
export {
|
||||
|
||||
@@ -23,8 +23,23 @@ export type {
|
||||
FeedbackTraceBundleFile,
|
||||
FeedbackTraceBundle,
|
||||
} from "./feedback.js";
|
||||
export type { InstanceExperimentalSettings, InstanceGeneralSettings, InstanceSettings, BackupRetentionPolicy } from "./instance.js";
|
||||
export { DAILY_RETENTION_PRESETS, WEEKLY_RETENTION_PRESETS, MONTHLY_RETENTION_PRESETS, DEFAULT_BACKUP_RETENTION } from "./instance.js";
|
||||
export type {
|
||||
InstanceExperimentalSettings,
|
||||
InstanceGeneralSettings,
|
||||
InstanceSettings,
|
||||
BackupRetentionPolicy,
|
||||
IssueGraphLivenessAutoRecoveryPreview,
|
||||
IssueGraphLivenessAutoRecoveryPreviewItem,
|
||||
} from "./instance.js";
|
||||
export {
|
||||
DAILY_RETENTION_PRESETS,
|
||||
WEEKLY_RETENTION_PRESETS,
|
||||
MONTHLY_RETENTION_PRESETS,
|
||||
DEFAULT_BACKUP_RETENTION,
|
||||
DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
MIN_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
MAX_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
} from "./instance.js";
|
||||
export type {
|
||||
CompanySkillSourceType,
|
||||
CompanySkillTrustLevel,
|
||||
|
||||
@@ -3,6 +3,9 @@ import type { FeedbackDataSharingPreference } from "./feedback.js";
|
||||
export const DAILY_RETENTION_PRESETS = [3, 7, 14] as const;
|
||||
export const WEEKLY_RETENTION_PRESETS = [1, 2, 4] as const;
|
||||
export const MONTHLY_RETENTION_PRESETS = [1, 3, 6] as const;
|
||||
export const DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS = 24;
|
||||
export const MIN_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS = 1;
|
||||
export const MAX_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS = 24 * 30;
|
||||
|
||||
export interface BackupRetentionPolicy {
|
||||
dailyDays: (typeof DAILY_RETENTION_PRESETS)[number];
|
||||
@@ -28,6 +31,7 @@ export interface InstanceExperimentalSettings {
|
||||
enableIsolatedWorkspaces: boolean;
|
||||
autoRestartDevServerWhenIdle: boolean;
|
||||
enableIssueGraphLivenessAutoRecovery: boolean;
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: number;
|
||||
}
|
||||
|
||||
export interface InstanceSettings {
|
||||
@@ -37,3 +41,34 @@ export interface InstanceSettings {
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
export interface IssueGraphLivenessAutoRecoveryPreviewItem {
|
||||
issueId: string;
|
||||
identifier: string | null;
|
||||
title: string;
|
||||
state: string;
|
||||
severity: string;
|
||||
reason: string;
|
||||
recoveryIssueId: string;
|
||||
recoveryIdentifier: string | null;
|
||||
recoveryTitle: string | null;
|
||||
recommendedOwnerAgentId: string | null;
|
||||
incidentKey: string;
|
||||
latestDependencyUpdatedAt: string;
|
||||
dependencyPath: Array<{
|
||||
issueId: string;
|
||||
identifier: string | null;
|
||||
title: string;
|
||||
status: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface IssueGraphLivenessAutoRecoveryPreview {
|
||||
lookbackHours: number;
|
||||
cutoff: string;
|
||||
generatedAt: string;
|
||||
findings: number;
|
||||
recoverableFindings: number;
|
||||
skippedOutsideLookback: number;
|
||||
items: IssueGraphLivenessAutoRecoveryPreviewItem[];
|
||||
}
|
||||
@@ -5,8 +5,10 @@ export {
|
||||
type PatchInstanceGeneralSettings,
|
||||
instanceExperimentalSettingsSchema,
|
||||
patchInstanceExperimentalSettingsSchema,
|
||||
issueGraphLivenessAutoRecoveryRequestSchema,
|
||||
type InstanceExperimentalSettings,
|
||||
type PatchInstanceExperimentalSettings,
|
||||
type IssueGraphLivenessAutoRecoveryRequest,
|
||||
} from "./instance.js";
|
||||
|
||||
export {
|
||||
|
||||
@@ -5,6 +5,9 @@ import {
|
||||
WEEKLY_RETENTION_PRESETS,
|
||||
MONTHLY_RETENTION_PRESETS,
|
||||
DEFAULT_BACKUP_RETENTION,
|
||||
DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
MAX_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
MIN_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
} from "../types/instance.js";
|
||||
import { feedbackDataSharingPreferenceSchema } from "./feedback.js";
|
||||
|
||||
@@ -36,12 +39,30 @@ export const instanceExperimentalSettingsSchema = z.object({
|
||||
enableEnvironments: z.boolean().default(false),
|
||||
enableIsolatedWorkspaces: z.boolean().default(false),
|
||||
autoRestartDevServerWhenIdle: z.boolean().default(false),
|
||||
enableIssueGraphLivenessAutoRecovery: z.boolean().default(false),
|
||||
enableIssueGraphLivenessAutoRecovery: z.boolean().default(true),
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: z
|
||||
.number()
|
||||
.int()
|
||||
.min(MIN_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS)
|
||||
.max(MAX_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS)
|
||||
.default(DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS),
|
||||
}).strict();
|
||||
|
||||
export const patchInstanceExperimentalSettingsSchema = instanceExperimentalSettingsSchema.partial();
|
||||
|
||||
export const issueGraphLivenessAutoRecoveryRequestSchema = z.object({
|
||||
lookbackHours: z
|
||||
.number()
|
||||
.int()
|
||||
.min(MIN_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS)
|
||||
.max(MAX_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS)
|
||||
.optional(),
|
||||
}).strict();
|
||||
|
||||
export type InstanceGeneralSettings = z.infer<typeof instanceGeneralSettingsSchema>;
|
||||
export type PatchInstanceGeneralSettings = z.infer<typeof patchInstanceGeneralSettingsSchema>;
|
||||
export type InstanceExperimentalSettings = z.infer<typeof instanceExperimentalSettingsSchema>;
|
||||
export type PatchInstanceExperimentalSettings = z.infer<typeof patchInstanceExperimentalSettingsSchema>;
|
||||
export type IssueGraphLivenessAutoRecoveryRequest = z.infer<
|
||||
typeof issueGraphLivenessAutoRecoveryRequestSchema
|
||||
>;
|
||||
@@ -103,6 +103,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
||||
await instanceSettingsService(db).updateExperimental({
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 24,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -116,7 +117,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
||||
});
|
||||
}
|
||||
|
||||
async function seedBlockedChain(opts: { stale?: boolean } = {}) {
|
||||
async function seedBlockedChain(opts: { outsideLookback?: boolean } = {}) {
|
||||
const companyId = randomUUID();
|
||||
const managerId = randomUUID();
|
||||
const coderId = randomUUID();
|
||||
@@ -157,9 +158,9 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
||||
},
|
||||
]);
|
||||
|
||||
const issueTimestamp = opts.stale === false
|
||||
? new Date()
|
||||
: new Date(Date.now() - 25 * 60 * 60 * 1000);
|
||||
const issueTimestamp = opts.outsideLookback === true
|
||||
? new Date(Date.now() - 25 * 60 * 60 * 1000)
|
||||
: new Date(Date.now() - 60 * 60 * 1000);
|
||||
await db.insert(issues).values([
|
||||
{
|
||||
id: blockedIssueId,
|
||||
@@ -197,6 +198,9 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
||||
}
|
||||
|
||||
it("keeps liveness findings advisory when auto recovery is disabled", async () => {
|
||||
await instanceSettingsService(db).updateExperimental({
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
});
|
||||
const { companyId } = await seedBlockedChain();
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
@@ -214,16 +218,16 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
||||
expect(escalations).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not create recovery issues until the dependency path is stale for 24 hours", async () => {
|
||||
it("does not create recovery issues outside the configured lookback window", async () => {
|
||||
await enableAutoRecovery();
|
||||
const { companyId } = await seedBlockedChain({ stale: false });
|
||||
const { companyId } = await seedBlockedChain({ outsideLookback: true });
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
const result = await heartbeat.reconcileIssueGraphLiveness();
|
||||
|
||||
expect(result.findings).toBe(1);
|
||||
expect(result.escalationsCreated).toBe(0);
|
||||
expect(result.skippedAutoRecoveryTooYoung).toBe(1);
|
||||
expect(result.skippedOutsideLookback).toBe(1);
|
||||
|
||||
const escalations = await db
|
||||
.select()
|
||||
@@ -424,7 +428,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
||||
const dependentExecutionWorkspaceId = randomUUID();
|
||||
const blockerExecutionWorkspaceId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const issueTimestamp = new Date(Date.now() - 25 * 60 * 60 * 1000);
|
||||
const issueTimestamp = new Date(Date.now() - 60 * 60 * 1000);
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
@@ -559,7 +563,7 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
|
||||
const { companyId, blockedIssueId, blockerIssueId } = await seedBlockedChain();
|
||||
const secondBlockedIssueId = randomUUID();
|
||||
const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
|
||||
const issueTimestamp = new Date(Date.now() - 25 * 60 * 60 * 1000);
|
||||
const issueTimestamp = new Date(Date.now() - 60 * 60 * 1000);
|
||||
await db.insert(issues).values({
|
||||
id: secondBlockedIssueId,
|
||||
companyId,
|
||||
|
||||
@@ -9,10 +9,15 @@ const mockInstanceSettingsService = vi.hoisted(() => ({
|
||||
updateExperimental: vi.fn(),
|
||||
listCompanyIds: vi.fn(),
|
||||
}));
|
||||
const mockHeartbeatService = vi.hoisted(() => ({
|
||||
buildIssueGraphLivenessAutoRecoveryPreview: vi.fn(),
|
||||
reconcileIssueGraphLiveness: vi.fn(),
|
||||
}));
|
||||
const mockLogActivity = vi.hoisted(() => vi.fn());
|
||||
|
||||
function registerModuleMocks() {
|
||||
vi.doMock("../services/index.js", () => ({
|
||||
heartbeatService: () => mockHeartbeatService,
|
||||
instanceSettingsService: () => mockInstanceSettingsService,
|
||||
logActivity: mockLogActivity,
|
||||
}));
|
||||
@@ -48,6 +53,8 @@ describe("instance settings routes", () => {
|
||||
mockInstanceSettingsService.updateGeneral.mockReset();
|
||||
mockInstanceSettingsService.updateExperimental.mockReset();
|
||||
mockInstanceSettingsService.listCompanyIds.mockReset();
|
||||
mockHeartbeatService.buildIssueGraphLivenessAutoRecoveryPreview.mockReset();
|
||||
mockHeartbeatService.reconcileIssueGraphLiveness.mockReset();
|
||||
mockLogActivity.mockReset();
|
||||
mockInstanceSettingsService.getGeneral.mockResolvedValue({
|
||||
censorUsernameInLogs: false,
|
||||
@@ -58,7 +65,8 @@ describe("instance settings routes", () => {
|
||||
enableEnvironments: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 24,
|
||||
});
|
||||
mockInstanceSettingsService.updateGeneral.mockResolvedValue({
|
||||
id: "instance-settings-1",
|
||||
@@ -74,10 +82,32 @@ describe("instance settings routes", () => {
|
||||
enableEnvironments: true,
|
||||
enableIsolatedWorkspaces: true,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 24,
|
||||
},
|
||||
});
|
||||
mockInstanceSettingsService.listCompanyIds.mockResolvedValue(["company-1", "company-2"]);
|
||||
mockHeartbeatService.buildIssueGraphLivenessAutoRecoveryPreview.mockResolvedValue({
|
||||
lookbackHours: 24,
|
||||
cutoff: "2026-04-26T12:00:00.000Z",
|
||||
generatedAt: "2026-04-27T12:00:00.000Z",
|
||||
findings: 1,
|
||||
recoverableFindings: 1,
|
||||
skippedOutsideLookback: 0,
|
||||
items: [],
|
||||
});
|
||||
mockHeartbeatService.reconcileIssueGraphLiveness.mockResolvedValue({
|
||||
findings: 1,
|
||||
autoRecoveryEnabled: true,
|
||||
lookbackHours: 24,
|
||||
cutoff: "2026-04-26T12:00:00.000Z",
|
||||
escalationsCreated: 1,
|
||||
existingEscalations: 0,
|
||||
skipped: 0,
|
||||
skippedAutoRecoveryDisabled: 0,
|
||||
skippedOutsideLookback: 0,
|
||||
escalationIssueIds: ["issue-2"],
|
||||
});
|
||||
});
|
||||
|
||||
it("allows local board users to read and update experimental settings", async () => {
|
||||
@@ -94,7 +124,8 @@ describe("instance settings routes", () => {
|
||||
enableEnvironments: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 24,
|
||||
});
|
||||
|
||||
const patchRes = await request(app)
|
||||
@@ -138,14 +169,58 @@ describe("instance settings routes", () => {
|
||||
|
||||
await request(app)
|
||||
.patch("/api/instance/settings/experimental")
|
||||
.send({ enableIssueGraphLivenessAutoRecovery: true })
|
||||
.send({
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 12,
|
||||
})
|
||||
.expect(200);
|
||||
|
||||
expect(mockInstanceSettingsService.updateExperimental).toHaveBeenCalledWith({
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: 12,
|
||||
});
|
||||
});
|
||||
|
||||
it("previews issue graph liveness recovery candidates before enabling", async () => {
|
||||
const app = await createApp({
|
||||
type: "board",
|
||||
userId: "local-board",
|
||||
source: "local_implicit",
|
||||
isInstanceAdmin: true,
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.post("/api/instance/settings/experimental/issue-graph-liveness-auto-recovery/preview")
|
||||
.send({ lookbackHours: 12 })
|
||||
.expect(200);
|
||||
|
||||
expect(res.body).toMatchObject({ lookbackHours: 24, recoverableFindings: 1 });
|
||||
expect(mockHeartbeatService.buildIssueGraphLivenessAutoRecoveryPreview).toHaveBeenCalledWith({
|
||||
lookbackHours: 12,
|
||||
});
|
||||
});
|
||||
|
||||
it("kicks off issue graph liveness recovery on demand", async () => {
|
||||
const app = await createApp({
|
||||
type: "board",
|
||||
userId: "local-board",
|
||||
source: "local_implicit",
|
||||
isInstanceAdmin: true,
|
||||
});
|
||||
|
||||
await request(app)
|
||||
.post("/api/instance/settings/experimental/issue-graph-liveness-auto-recovery/run")
|
||||
.send({ lookbackHours: 12 })
|
||||
.expect(200);
|
||||
|
||||
expect(mockHeartbeatService.reconcileIssueGraphLiveness).toHaveBeenCalledWith({
|
||||
runId: null,
|
||||
force: true,
|
||||
lookbackHours: 12,
|
||||
});
|
||||
expect(mockLogActivity).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("allows local board users to update environment controls", async () => {
|
||||
const app = await createApp({
|
||||
type: "board",
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
import { Router, type Request } from "express";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { patchInstanceExperimentalSettingsSchema, patchInstanceGeneralSettingsSchema } from "@paperclipai/shared";
|
||||
import {
|
||||
issueGraphLivenessAutoRecoveryRequestSchema,
|
||||
patchInstanceExperimentalSettingsSchema,
|
||||
patchInstanceGeneralSettingsSchema,
|
||||
} from "@paperclipai/shared";
|
||||
import { forbidden } from "../errors.js";
|
||||
import { validate } from "../middleware/validate.js";
|
||||
import { instanceSettingsService, logActivity } from "../services/index.js";
|
||||
import { heartbeatService, instanceSettingsService, logActivity } from "../services/index.js";
|
||||
import { assertBoardOrgAccess, getActorInfo } from "./authz.js";
|
||||
|
||||
function assertCanManageInstanceSettings(req: Request) {
|
||||
@@ -19,6 +23,7 @@ function assertCanManageInstanceSettings(req: Request) {
|
||||
export function instanceSettingsRoutes(db: Db) {
|
||||
const router = Router();
|
||||
const svc = instanceSettingsService(db);
|
||||
const heartbeat = heartbeatService(db);
|
||||
|
||||
router.get("/instance/settings/general", async (req, res) => {
|
||||
// General settings (e.g. keyboardShortcuts) are readable by any
|
||||
@@ -94,5 +99,53 @@ export function instanceSettingsRoutes(db: Db) {
|
||||
},
|
||||
);
|
||||
|
||||
router.post(
|
||||
"/instance/settings/experimental/issue-graph-liveness-auto-recovery/preview",
|
||||
validate(issueGraphLivenessAutoRecoveryRequestSchema),
|
||||
async (req, res) => {
|
||||
assertCanManageInstanceSettings(req);
|
||||
res.json(await heartbeat.buildIssueGraphLivenessAutoRecoveryPreview({
|
||||
lookbackHours: req.body.lookbackHours,
|
||||
}));
|
||||
},
|
||||
);
|
||||
|
||||
router.post(
|
||||
"/instance/settings/experimental/issue-graph-liveness-auto-recovery/run",
|
||||
validate(issueGraphLivenessAutoRecoveryRequestSchema),
|
||||
async (req, res) => {
|
||||
assertCanManageInstanceSettings(req);
|
||||
const actor = getActorInfo(req);
|
||||
const result = await heartbeat.reconcileIssueGraphLiveness({
|
||||
runId: actor.runId,
|
||||
force: true,
|
||||
lookbackHours: req.body.lookbackHours,
|
||||
});
|
||||
const companyIds = await svc.listCompanyIds();
|
||||
await Promise.all(
|
||||
companyIds.map((companyId) =>
|
||||
logActivity(db, {
|
||||
companyId,
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
agentId: actor.agentId,
|
||||
runId: actor.runId,
|
||||
action: "instance.settings.issue_graph_liveness_auto_recovery_run",
|
||||
entityType: "instance_settings",
|
||||
entityId: "default",
|
||||
details: {
|
||||
lookbackHours: result.lookbackHours,
|
||||
escalationsCreated: result.escalationsCreated,
|
||||
existingEscalations: result.existingEscalations,
|
||||
skippedOutsideLookback: result.skippedOutsideLookback,
|
||||
escalationIssueIds: result.escalationIssueIds,
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
res.json(result);
|
||||
},
|
||||
);
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -4502,7 +4502,15 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
return recovery.buildRunOutputSilence(run, now);
|
||||
}
|
||||
|
||||
async function reconcileIssueGraphLiveness(opts?: { runId?: string | null }) {
|
||||
async function buildIssueGraphLivenessAutoRecoveryPreview(opts?: { lookbackHours?: number; now?: Date }) {
|
||||
return recovery.buildIssueGraphLivenessAutoRecoveryPreview(opts);
|
||||
}
|
||||
|
||||
async function reconcileIssueGraphLiveness(opts?: {
|
||||
runId?: string | null;
|
||||
force?: boolean;
|
||||
lookbackHours?: number;
|
||||
}) {
|
||||
return recovery.reconcileIssueGraphLiveness(opts);
|
||||
}
|
||||
|
||||
@@ -7477,6 +7485,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
|
||||
reconcileStrandedAssignedIssues,
|
||||
|
||||
buildIssueGraphLivenessAutoRecoveryPreview,
|
||||
|
||||
reconcileIssueGraphLiveness,
|
||||
|
||||
scanSilentActiveRuns,
|
||||
|
||||
@@ -3,6 +3,7 @@ import { companies, instanceSettings } from "@paperclipai/db";
|
||||
import {
|
||||
DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE,
|
||||
DEFAULT_BACKUP_RETENTION,
|
||||
DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
instanceGeneralSettingsSchema,
|
||||
type InstanceGeneralSettings,
|
||||
instanceExperimentalSettingsSchema,
|
||||
@@ -41,14 +42,19 @@ function normalizeExperimentalSettings(raw: unknown): InstanceExperimentalSettin
|
||||
enableEnvironments: parsed.data.enableEnvironments ?? false,
|
||||
enableIsolatedWorkspaces: parsed.data.enableIsolatedWorkspaces ?? false,
|
||||
autoRestartDevServerWhenIdle: parsed.data.autoRestartDevServerWhenIdle ?? false,
|
||||
enableIssueGraphLivenessAutoRecovery: parsed.data.enableIssueGraphLivenessAutoRecovery ?? false,
|
||||
enableIssueGraphLivenessAutoRecovery: parsed.data.enableIssueGraphLivenessAutoRecovery ?? true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours:
|
||||
parsed.data.issueGraphLivenessAutoRecoveryLookbackHours ??
|
||||
DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
};
|
||||
}
|
||||
return {
|
||||
enableEnvironments: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
autoRestartDevServerWhenIdle: false,
|
||||
enableIssueGraphLivenessAutoRecovery: false,
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours:
|
||||
DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
import { and, asc, desc, eq, gt, inArray, isNull, notInArray, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
MAX_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
MIN_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
type IssueGraphLivenessAutoRecoveryPreview,
|
||||
type IssueGraphLivenessAutoRecoveryPreviewItem,
|
||||
} from "@paperclipai/shared";
|
||||
import {
|
||||
agents,
|
||||
agentWakeupRequests,
|
||||
@@ -38,7 +45,6 @@ import { isAutomaticRecoverySuppressedByPauseHold } from "./pause-hold-guard.js"
|
||||
|
||||
const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;
|
||||
const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "timed_out"] as const;
|
||||
const ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_MIN_STALE_MS = 24 * 60 * 60 * 1000;
|
||||
export const ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS = 60 * 60 * 1000;
|
||||
export const ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS = 4 * 60 * 60 * 1000;
|
||||
export const ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS = 30 * 60 * 1000;
|
||||
@@ -1857,18 +1863,86 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
return result;
|
||||
}
|
||||
|
||||
async function isLivenessFindingOldEnoughForAutoRecovery(finding: IssueLivenessFinding, now = new Date()) {
|
||||
function normalizeIssueGraphLivenessAutoRecoveryLookbackHours(raw: unknown) {
|
||||
const numeric = Math.floor(asNumber(raw, DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS));
|
||||
return Math.min(
|
||||
MAX_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS,
|
||||
Math.max(MIN_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS, numeric),
|
||||
);
|
||||
}
|
||||
|
||||
async function latestDependencyUpdatedAtForLivenessFinding(finding: IssueLivenessFinding) {
|
||||
const issueIds = [...new Set(finding.dependencyPath.map((entry) => entry.issueId))];
|
||||
if (issueIds.length === 0) return false;
|
||||
if (issueIds.length === 0) return null;
|
||||
const rows = await db
|
||||
.select({ id: issues.id, updatedAt: issues.updatedAt })
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, finding.companyId), inArray(issues.id, issueIds)));
|
||||
if (rows.length !== issueIds.length) return false;
|
||||
const latestUpdatedAt = rows.reduce((latest, row) =>
|
||||
if (rows.length !== issueIds.length) return null;
|
||||
return rows.reduce((latest, row) =>
|
||||
row.updatedAt > latest ? row.updatedAt : latest,
|
||||
rows[0]!.updatedAt);
|
||||
return now.getTime() - latestUpdatedAt.getTime() >= ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_MIN_STALE_MS;
|
||||
}
|
||||
|
||||
async function isLivenessFindingInsideAutoRecoveryLookback(
|
||||
finding: IssueLivenessFinding,
|
||||
cutoff: Date,
|
||||
) {
|
||||
const latestUpdatedAt = await latestDependencyUpdatedAtForLivenessFinding(finding);
|
||||
return Boolean(latestUpdatedAt && latestUpdatedAt >= cutoff);
|
||||
}
|
||||
|
||||
async function buildIssueGraphLivenessAutoRecoveryPreview(
|
||||
opts?: { lookbackHours?: number; now?: Date },
|
||||
): Promise<IssueGraphLivenessAutoRecoveryPreview> {
|
||||
const now = opts?.now ?? new Date();
|
||||
const lookbackHours = normalizeIssueGraphLivenessAutoRecoveryLookbackHours(opts?.lookbackHours);
|
||||
const cutoff = new Date(now.getTime() - lookbackHours * 60 * 60 * 1000);
|
||||
const findings = await collectIssueGraphLivenessFindings();
|
||||
const issueIds = [...new Set(findings.map((finding) => finding.recoveryIssueId))];
|
||||
const recoveryRows = issueIds.length > 0
|
||||
? await db
|
||||
.select({ id: issues.id, identifier: issues.identifier, title: issues.title })
|
||||
.from(issues)
|
||||
.where(inArray(issues.id, issueIds))
|
||||
: [];
|
||||
const recoveryById = new Map(recoveryRows.map((row) => [row.id, row]));
|
||||
const items: IssueGraphLivenessAutoRecoveryPreviewItem[] = [];
|
||||
let skippedOutsideLookback = 0;
|
||||
|
||||
for (const finding of findings) {
|
||||
const latestDependencyUpdatedAt = await latestDependencyUpdatedAtForLivenessFinding(finding);
|
||||
if (!latestDependencyUpdatedAt || latestDependencyUpdatedAt < cutoff) {
|
||||
skippedOutsideLookback += 1;
|
||||
continue;
|
||||
}
|
||||
const recoveryIssue = recoveryById.get(finding.recoveryIssueId);
|
||||
items.push({
|
||||
issueId: finding.issueId,
|
||||
identifier: finding.identifier,
|
||||
title: finding.dependencyPath[0]?.title ?? finding.identifier ?? finding.issueId,
|
||||
state: finding.state,
|
||||
severity: finding.severity,
|
||||
reason: finding.reason,
|
||||
recoveryIssueId: finding.recoveryIssueId,
|
||||
recoveryIdentifier: recoveryIssue?.identifier ?? null,
|
||||
recoveryTitle: recoveryIssue?.title ?? null,
|
||||
recommendedOwnerAgentId: finding.recommendedOwnerAgentId,
|
||||
incidentKey: finding.incidentKey,
|
||||
latestDependencyUpdatedAt: latestDependencyUpdatedAt.toISOString(),
|
||||
dependencyPath: finding.dependencyPath,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
lookbackHours,
|
||||
cutoff: cutoff.toISOString(),
|
||||
generatedAt: now.toISOString(),
|
||||
findings: findings.length,
|
||||
recoverableFindings: items.length,
|
||||
skippedOutsideLookback,
|
||||
items,
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveEscalationOwnerAgentId(
|
||||
@@ -2131,22 +2205,33 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
return { kind: "created" as const, escalationIssueId: escalation.id };
|
||||
}
|
||||
|
||||
async function reconcileIssueGraphLiveness(opts?: { runId?: string | null }) {
|
||||
async function reconcileIssueGraphLiveness(opts?: {
|
||||
runId?: string | null;
|
||||
force?: boolean;
|
||||
lookbackHours?: number;
|
||||
}) {
|
||||
const findings = await collectIssueGraphLivenessFindings();
|
||||
const experimentalSettings = await instanceSettings.getExperimental();
|
||||
const autoRecoveryEnabled = asBoolean(
|
||||
experimentalSettings.enableIssueGraphLivenessAutoRecovery,
|
||||
false,
|
||||
true,
|
||||
) || opts?.force === true;
|
||||
const lookbackHours = normalizeIssueGraphLivenessAutoRecoveryLookbackHours(
|
||||
opts?.lookbackHours ?? experimentalSettings.issueGraphLivenessAutoRecoveryLookbackHours,
|
||||
);
|
||||
const now = new Date();
|
||||
const cutoff = new Date(now.getTime() - lookbackHours * 60 * 60 * 1000);
|
||||
const obsoleteRecoveryCleanup = await retireObsoleteLivenessRecoveryIssues(findings);
|
||||
const result = {
|
||||
findings: findings.length,
|
||||
autoRecoveryEnabled,
|
||||
lookbackHours,
|
||||
cutoff: cutoff.toISOString(),
|
||||
escalationsCreated: 0,
|
||||
existingEscalations: 0,
|
||||
skipped: 0,
|
||||
skippedAutoRecoveryDisabled: 0,
|
||||
skippedAutoRecoveryTooYoung: 0,
|
||||
skippedOutsideLookback: 0,
|
||||
obsoleteRecoveriesRetired: obsoleteRecoveryCleanup.retired,
|
||||
obsoleteRecoveriesActiveSkipped: obsoleteRecoveryCleanup.activeSkipped,
|
||||
obsoleteRecoveryBlockerRelationsRemoved: obsoleteRecoveryCleanup.blockerRelationsRemoved,
|
||||
@@ -2160,10 +2245,9 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
return result;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
for (const finding of findings) {
|
||||
if (!await isLivenessFindingOldEnoughForAutoRecovery(finding, now)) {
|
||||
result.skippedAutoRecoveryTooYoung += 1;
|
||||
if (!await isLivenessFindingInsideAutoRecoveryLookback(finding, cutoff)) {
|
||||
result.skippedOutsideLookback += 1;
|
||||
result.skipped += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -2197,6 +2281,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
recordWatchdogDecision,
|
||||
scanSilentActiveRuns,
|
||||
reconcileStrandedAssignedIssues,
|
||||
buildIssueGraphLivenessAutoRecoveryPreview,
|
||||
reconcileIssueGraphLiveness,
|
||||
readRecoveryTimerIntervalMs,
|
||||
};
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type {
|
||||
InstanceExperimentalSettings,
|
||||
InstanceGeneralSettings,
|
||||
IssueGraphLivenessAutoRecoveryPreview,
|
||||
PatchInstanceGeneralSettings,
|
||||
PatchInstanceExperimentalSettings,
|
||||
} from "@paperclipai/shared";
|
||||
@@ -15,4 +16,25 @@ export const instanceSettingsApi = {
|
||||
api.get<InstanceExperimentalSettings>("/instance/settings/experimental"),
|
||||
updateExperimental: (patch: PatchInstanceExperimentalSettings) =>
|
||||
api.patch<InstanceExperimentalSettings>("/instance/settings/experimental", patch),
|
||||
previewIssueGraphLivenessAutoRecovery: (input: { lookbackHours?: number }) =>
|
||||
api.post<IssueGraphLivenessAutoRecoveryPreview>(
|
||||
"/instance/settings/experimental/issue-graph-liveness-auto-recovery/preview",
|
||||
input,
|
||||
),
|
||||
runIssueGraphLivenessAutoRecovery: (input: { lookbackHours?: number }) =>
|
||||
api.post<{
|
||||
findings: number;
|
||||
autoRecoveryEnabled: boolean;
|
||||
lookbackHours: number;
|
||||
cutoff: string;
|
||||
escalationsCreated: number;
|
||||
existingEscalations: number;
|
||||
skipped: number;
|
||||
skippedAutoRecoveryDisabled: number;
|
||||
skippedOutsideLookback: number;
|
||||
escalationIssueIds: string[];
|
||||
}>(
|
||||
"/instance/settings/experimental/issue-graph-liveness-auto-recovery/run",
|
||||
input,
|
||||
),
|
||||
};
|
||||
@@ -1,16 +1,130 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { FlaskConical } from "lucide-react";
|
||||
import type { PatchInstanceExperimentalSettings } from "@paperclipai/shared";
|
||||
import { Clock, FlaskConical, Play, Search } from "lucide-react";
|
||||
import type {
|
||||
IssueGraphLivenessAutoRecoveryPreview,
|
||||
PatchInstanceExperimentalSettings,
|
||||
} from "@paperclipai/shared";
|
||||
import { instanceSettingsApi } from "@/api/instanceSettings";
|
||||
import { useBreadcrumbs } from "../context/BreadcrumbContext";
|
||||
import { queryKeys } from "../lib/queryKeys";
|
||||
import { ToggleSwitch } from "@/components/ui/toggle-switch";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from "@/components/ui/dialog";
|
||||
|
||||
function issueHref(identifier: string | null, issueId: string) {
|
||||
if (!identifier) return `/issues/${issueId}`;
|
||||
const prefix = identifier.split("-")[0] || "PAP";
|
||||
return `/${prefix}/issues/${identifier}`;
|
||||
}
|
||||
|
||||
function formatRecoveryState(state: string) {
|
||||
return state.replace(/_/g, " ");
|
||||
}
|
||||
|
||||
function RecoveryPreviewDialog({
|
||||
preview,
|
||||
open,
|
||||
onOpenChange,
|
||||
onEnableOnly,
|
||||
onEnableAndRun,
|
||||
isPending,
|
||||
}: {
|
||||
preview: IssueGraphLivenessAutoRecoveryPreview | null;
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
onEnableOnly: () => void;
|
||||
onEnableAndRun: () => void;
|
||||
isPending: boolean;
|
||||
}) {
|
||||
const count = preview?.recoverableFindings ?? 0;
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="sm:max-w-3xl">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Confirm auto-recovery</DialogTitle>
|
||||
<DialogDescription>
|
||||
{preview
|
||||
? `${count} recovery ${count === 1 ? "task" : "tasks"} match the last ${preview.lookbackHours} hours.`
|
||||
: "Checking recovery candidates before enabling."}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="max-h-[min(28rem,65vh)] space-y-3 overflow-y-auto pr-1">
|
||||
{preview && preview.items.length === 0 ? (
|
||||
<div className="rounded-md border border-border bg-muted/30 px-3 py-4 text-sm text-muted-foreground">
|
||||
No recovery tasks would be created right now. Auto-recovery can still run for future liveness incidents in
|
||||
this window.
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{preview?.items.map((item) => (
|
||||
<div key={item.incidentKey} className="rounded-md border border-border bg-card px-3 py-3">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<a
|
||||
href={issueHref(item.identifier, item.issueId)}
|
||||
className="text-sm font-medium text-primary underline-offset-2 hover:underline"
|
||||
>
|
||||
{item.identifier ?? item.issueId}
|
||||
</a>
|
||||
<span className="rounded-sm bg-muted px-1.5 py-0.5 text-xs text-muted-foreground">
|
||||
{formatRecoveryState(item.state)}
|
||||
</span>
|
||||
</div>
|
||||
<p className="mt-1 text-sm text-foreground">{item.title}</p>
|
||||
<p className="mt-1 text-xs text-muted-foreground">{item.reason}</p>
|
||||
<div className="mt-2 text-xs text-muted-foreground">
|
||||
Recovery target:{" "}
|
||||
<a
|
||||
href={issueHref(item.recoveryIdentifier, item.recoveryIssueId)}
|
||||
className="text-primary underline-offset-2 hover:underline"
|
||||
>
|
||||
{item.recoveryIdentifier ?? item.recoveryIssueId}
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{preview && preview.skippedOutsideLookback > 0 ? (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{preview.skippedOutsideLookback} current{" "}
|
||||
{preview.skippedOutsideLookback === 1 ? "finding is" : "findings are"} outside the configured lookback and
|
||||
will not be touched.
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
<DialogFooter>
|
||||
<Button variant="outline" onClick={() => onOpenChange(false)} disabled={isPending}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button variant="outline" onClick={onEnableOnly} disabled={isPending || !preview}>
|
||||
Enable only
|
||||
</Button>
|
||||
<Button onClick={onEnableAndRun} disabled={isPending || !preview}>
|
||||
{count > 0 ? `Enable and create ${count}` : "Enable"}
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
|
||||
export function InstanceExperimentalSettings() {
|
||||
const { setBreadcrumbs } = useBreadcrumbs();
|
||||
const queryClient = useQueryClient();
|
||||
const [actionError, setActionError] = useState<string | null>(null);
|
||||
const [lookbackHoursDraft, setLookbackHoursDraft] = useState("24");
|
||||
const [previewDialogOpen, setPreviewDialogOpen] = useState(false);
|
||||
const [pendingPreview, setPendingPreview] = useState<IssueGraphLivenessAutoRecoveryPreview | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
setBreadcrumbs([
|
||||
@@ -39,6 +153,42 @@ export function InstanceExperimentalSettings() {
|
||||
},
|
||||
});
|
||||
|
||||
const previewMutation = useMutation({
|
||||
mutationFn: async (lookbackHours: number) =>
|
||||
instanceSettingsApi.previewIssueGraphLivenessAutoRecovery({ lookbackHours }),
|
||||
onSuccess: (preview) => {
|
||||
setActionError(null);
|
||||
setPendingPreview(preview);
|
||||
setPreviewDialogOpen(true);
|
||||
},
|
||||
onError: (error) => {
|
||||
setActionError(error instanceof Error ? error.message : "Failed to preview recovery tasks.");
|
||||
},
|
||||
});
|
||||
|
||||
const runRecoveryMutation = useMutation({
|
||||
mutationFn: async (lookbackHours: number) =>
|
||||
instanceSettingsApi.runIssueGraphLivenessAutoRecovery({ lookbackHours }),
|
||||
onSuccess: async () => {
|
||||
setActionError(null);
|
||||
setPreviewDialogOpen(false);
|
||||
await Promise.all([
|
||||
queryClient.invalidateQueries({ queryKey: queryKeys.instance.experimentalSettings }),
|
||||
queryClient.invalidateQueries({ queryKey: queryKeys.health }),
|
||||
]);
|
||||
},
|
||||
onError: (error) => {
|
||||
setActionError(error instanceof Error ? error.message : "Failed to create recovery tasks.");
|
||||
},
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
const next = experimentalQuery.data?.issueGraphLivenessAutoRecoveryLookbackHours;
|
||||
if (typeof next === "number") {
|
||||
setLookbackHoursDraft(String(next));
|
||||
}
|
||||
}, [experimentalQuery.data?.issueGraphLivenessAutoRecoveryLookbackHours]);
|
||||
|
||||
if (experimentalQuery.isLoading) {
|
||||
return <div className="text-sm text-muted-foreground">Loading experimental settings...</div>;
|
||||
}
|
||||
@@ -58,6 +208,41 @@ export function InstanceExperimentalSettings() {
|
||||
const autoRestartDevServerWhenIdle = experimentalQuery.data?.autoRestartDevServerWhenIdle === true;
|
||||
const enableIssueGraphLivenessAutoRecovery =
|
||||
experimentalQuery.data?.enableIssueGraphLivenessAutoRecovery === true;
|
||||
const lookbackHours =
|
||||
experimentalQuery.data?.issueGraphLivenessAutoRecoveryLookbackHours ?? 24;
|
||||
const parsedLookbackHours = Number.parseInt(lookbackHoursDraft, 10);
|
||||
const lookbackHoursIsValid =
|
||||
Number.isInteger(parsedLookbackHours) && parsedLookbackHours >= 1 && parsedLookbackHours <= 720;
|
||||
const recoveryActionPending =
|
||||
toggleMutation.isPending || previewMutation.isPending || runRecoveryMutation.isPending;
|
||||
|
||||
function previewForEnable() {
|
||||
if (!lookbackHoursIsValid) {
|
||||
setActionError("Lookback hours must be a whole number from 1 to 720.");
|
||||
return;
|
||||
}
|
||||
previewMutation.mutate(parsedLookbackHours);
|
||||
}
|
||||
|
||||
function enableOnly() {
|
||||
if (!lookbackHoursIsValid) return;
|
||||
toggleMutation.mutate({
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: parsedLookbackHours,
|
||||
}, {
|
||||
onSuccess: () => setPreviewDialogOpen(false),
|
||||
});
|
||||
}
|
||||
|
||||
function enableAndRun() {
|
||||
if (!lookbackHoursIsValid) return;
|
||||
toggleMutation.mutate({
|
||||
enableIssueGraphLivenessAutoRecovery: true,
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: parsedLookbackHours,
|
||||
}, {
|
||||
onSuccess: () => runRecoveryMutation.mutate(parsedLookbackHours),
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="max-w-4xl space-y-6">
|
||||
@@ -132,26 +317,99 @@ export function InstanceExperimentalSettings() {
|
||||
</section>
|
||||
|
||||
<section className="rounded-xl border border-border bg-card p-5">
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1.5">
|
||||
<h2 className="text-sm font-semibold">Auto-Create Issue Recovery Tasks</h2>
|
||||
<p className="max-w-2xl text-sm text-muted-foreground">
|
||||
Let the heartbeat scheduler create recovery issues for issue dependency chains that have been stalled for
|
||||
at least 24 hours.
|
||||
</p>
|
||||
<div className="flex flex-col gap-5">
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="space-y-1.5">
|
||||
<h2 className="text-sm font-semibold">Auto-Create Issue Recovery Tasks</h2>
|
||||
<p className="max-w-2xl text-sm text-muted-foreground">
|
||||
Let the heartbeat scheduler create recovery issues for issue dependency chains found inside the
|
||||
configured lookback window.
|
||||
</p>
|
||||
</div>
|
||||
<ToggleSwitch
|
||||
checked={enableIssueGraphLivenessAutoRecovery}
|
||||
onCheckedChange={() => {
|
||||
if (enableIssueGraphLivenessAutoRecovery) {
|
||||
toggleMutation.mutate({ enableIssueGraphLivenessAutoRecovery: false });
|
||||
return;
|
||||
}
|
||||
previewForEnable();
|
||||
}}
|
||||
disabled={recoveryActionPending}
|
||||
aria-label="Toggle issue graph liveness auto-recovery"
|
||||
/>
|
||||
</div>
|
||||
<ToggleSwitch
|
||||
checked={enableIssueGraphLivenessAutoRecovery}
|
||||
onCheckedChange={() =>
|
||||
toggleMutation.mutate({
|
||||
enableIssueGraphLivenessAutoRecovery: !enableIssueGraphLivenessAutoRecovery,
|
||||
})
|
||||
}
|
||||
disabled={toggleMutation.isPending}
|
||||
aria-label="Toggle issue graph liveness auto-recovery"
|
||||
/>
|
||||
|
||||
<div className="grid gap-3 sm:grid-cols-[minmax(10rem,14rem)_1fr] sm:items-end">
|
||||
<label className="space-y-1.5">
|
||||
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
|
||||
<Clock className="h-3.5 w-3.5" />
|
||||
Lookback hours
|
||||
</span>
|
||||
<Input
|
||||
type="number"
|
||||
min={1}
|
||||
max={720}
|
||||
step={1}
|
||||
value={lookbackHoursDraft}
|
||||
onChange={(event) => setLookbackHoursDraft(event.target.value)}
|
||||
aria-invalid={!lookbackHoursIsValid}
|
||||
/>
|
||||
</label>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => {
|
||||
if (!lookbackHoursIsValid) {
|
||||
setActionError("Lookback hours must be a whole number from 1 to 720.");
|
||||
return;
|
||||
}
|
||||
toggleMutation.mutate({
|
||||
issueGraphLivenessAutoRecoveryLookbackHours: parsedLookbackHours,
|
||||
});
|
||||
}}
|
||||
disabled={recoveryActionPending || parsedLookbackHours === lookbackHours}
|
||||
>
|
||||
Save hours
|
||||
</Button>
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={previewForEnable}
|
||||
disabled={recoveryActionPending}
|
||||
>
|
||||
<Search className="h-4 w-4" />
|
||||
Preview
|
||||
</Button>
|
||||
<Button
|
||||
onClick={() => {
|
||||
if (!lookbackHoursIsValid) {
|
||||
setActionError("Lookback hours must be a whole number from 1 to 720.");
|
||||
return;
|
||||
}
|
||||
runRecoveryMutation.mutate(parsedLookbackHours);
|
||||
}}
|
||||
disabled={recoveryActionPending || !enableIssueGraphLivenessAutoRecovery}
|
||||
>
|
||||
<Play className="h-4 w-4" />
|
||||
Run now
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Current window: last {lookbackHours} {lookbackHours === 1 ? "hour" : "hours"}.
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<RecoveryPreviewDialog
|
||||
open={previewDialogOpen}
|
||||
onOpenChange={setPreviewDialogOpen}
|
||||
preview={pendingPreview}
|
||||
onEnableOnly={enableOnly}
|
||||
onEnableAndRun={enableAndRun}
|
||||
isPending={recoveryActionPending}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user