diff --git a/packages/db/src/migrations/0084_issue_recovery_actions.sql b/packages/db/src/migrations/0084_issue_recovery_actions.sql new file mode 100644 index 00000000000..6bce0bf519c --- /dev/null +++ b/packages/db/src/migrations/0084_issue_recovery_actions.sql @@ -0,0 +1,64 @@ +CREATE TABLE IF NOT EXISTS "issue_recovery_actions" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "company_id" uuid NOT NULL, + "source_issue_id" uuid NOT NULL, + "recovery_issue_id" uuid, + "kind" text NOT NULL, + "status" text DEFAULT 'active' NOT NULL, + "owner_type" text DEFAULT 'agent' NOT NULL, + "owner_agent_id" uuid, + "owner_user_id" text, + "previous_owner_agent_id" uuid, + "return_owner_agent_id" uuid, + "cause" text NOT NULL, + "fingerprint" text NOT NULL, + "evidence" jsonb DEFAULT '{}'::jsonb NOT NULL, + "next_action" text NOT NULL, + "wake_policy" jsonb, + "monitor_policy" jsonb, + "attempt_count" integer DEFAULT 0 NOT NULL, + "max_attempts" integer, + "timeout_at" timestamp with time zone, + "last_attempt_at" timestamp with time zone, + "outcome" text, + "resolution_note" text, + "resolved_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'issue_recovery_actions_company_id_companies_id_fk') THEN + ALTER TABLE "issue_recovery_actions" ADD CONSTRAINT "issue_recovery_actions_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE no action ON UPDATE no action; + END IF; +END $$;--> statement-breakpoint +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'issue_recovery_actions_source_issue_id_issues_id_fk') THEN + ALTER TABLE "issue_recovery_actions" ADD CONSTRAINT "issue_recovery_actions_source_issue_id_issues_id_fk" FOREIGN KEY ("source_issue_id") REFERENCES "public"."issues"("id") ON DELETE cascade ON UPDATE no action; + END IF; +END $$;--> statement-breakpoint +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'issue_recovery_actions_recovery_issue_id_issues_id_fk') THEN + ALTER TABLE "issue_recovery_actions" ADD CONSTRAINT "issue_recovery_actions_recovery_issue_id_issues_id_fk" FOREIGN KEY ("recovery_issue_id") REFERENCES "public"."issues"("id") ON DELETE set null ON UPDATE no action; + END IF; +END $$;--> statement-breakpoint +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'issue_recovery_actions_owner_agent_id_agents_id_fk') THEN + ALTER TABLE "issue_recovery_actions" ADD CONSTRAINT "issue_recovery_actions_owner_agent_id_agents_id_fk" FOREIGN KEY ("owner_agent_id") REFERENCES "public"."agents"("id") ON DELETE set null ON UPDATE no action; + END IF; +END $$;--> statement-breakpoint +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'issue_recovery_actions_previous_owner_agent_id_agents_id_fk') THEN + ALTER TABLE "issue_recovery_actions" ADD CONSTRAINT "issue_recovery_actions_previous_owner_agent_id_agents_id_fk" FOREIGN KEY ("previous_owner_agent_id") REFERENCES "public"."agents"("id") ON DELETE set null ON UPDATE no action; + END IF; +END $$;--> statement-breakpoint +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'issue_recovery_actions_return_owner_agent_id_agents_id_fk') THEN + ALTER TABLE "issue_recovery_actions" ADD CONSTRAINT "issue_recovery_actions_return_owner_agent_id_agents_id_fk" FOREIGN KEY ("return_owner_agent_id") REFERENCES "public"."agents"("id") ON DELETE set null ON UPDATE no action; + END IF; +END $$;--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "issue_recovery_actions_company_source_status_idx" ON "issue_recovery_actions" USING btree ("company_id","source_issue_id","status");--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "issue_recovery_actions_company_owner_status_idx" ON "issue_recovery_actions" USING btree ("company_id","owner_agent_id","status");--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "issue_recovery_actions_company_recovery_issue_idx" ON "issue_recovery_actions" USING btree ("company_id","recovery_issue_id");--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "issue_recovery_actions_active_source_uq" ON "issue_recovery_actions" USING btree ("company_id","source_issue_id") WHERE "issue_recovery_actions"."status" in ('active', 'escalated');--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS "issue_recovery_actions_active_fingerprint_uq" ON "issue_recovery_actions" USING btree ("company_id","source_issue_id","cause","fingerprint") WHERE "issue_recovery_actions"."status" in ('active', 'escalated'); diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 74214acd958..9d124c7d6d5 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -589,6 +589,13 @@ "when": 1778074536410, "tag": "0083_company_secret_provider_configs", "breakpoints": true + }, + { + "idx": 84, + "version": "7", + "when": 1778355326070, + "tag": "0084_issue_recovery_actions", + "breakpoints": true } ] } diff --git a/packages/db/src/schema/index.ts b/packages/db/src/schema/index.ts index 9099f9044c6..fdf5f7cb92a 100644 --- a/packages/db/src/schema/index.ts +++ b/packages/db/src/schema/index.ts @@ -29,6 +29,7 @@ export { workspaceRuntimeServices } from "./workspace_runtime_services.js"; export { projectGoals } from "./project_goals.js"; export { goals } from "./goals.js"; export { issues } from "./issues.js"; +export { issueRecoveryActions } from "./issue_recovery_actions.js"; export { issueReferenceMentions } from "./issue_reference_mentions.js"; export { issueRelations } from "./issue_relations.js"; export { routines, routineRevisions, routineTriggers, routineRuns } from "./routines.js"; diff --git a/packages/db/src/schema/issue_recovery_actions.ts b/packages/db/src/schema/issue_recovery_actions.ts new file mode 100644 index 00000000000..9c65a648de3 --- /dev/null +++ b/packages/db/src/schema/issue_recovery_actions.ts @@ -0,0 +1,68 @@ +import { sql } from "drizzle-orm"; +import { + index, + integer, + jsonb, + pgTable, + text, + timestamp, + uniqueIndex, + uuid, +} from "drizzle-orm/pg-core"; +import { agents } from "./agents.js"; +import { companies } from "./companies.js"; +import { issues } from "./issues.js"; + +export const issueRecoveryActions = pgTable( + "issue_recovery_actions", + { + id: uuid("id").primaryKey().defaultRandom(), + companyId: uuid("company_id").notNull().references(() => companies.id), + sourceIssueId: uuid("source_issue_id").notNull().references(() => issues.id, { onDelete: "cascade" }), + recoveryIssueId: uuid("recovery_issue_id").references(() => issues.id, { onDelete: "set null" }), + kind: text("kind").notNull(), + status: text("status").notNull().default("active"), + ownerType: text("owner_type").notNull().default("agent"), + ownerAgentId: uuid("owner_agent_id").references(() => agents.id, { onDelete: "set null" }), + ownerUserId: text("owner_user_id"), + previousOwnerAgentId: uuid("previous_owner_agent_id").references(() => agents.id, { onDelete: "set null" }), + returnOwnerAgentId: uuid("return_owner_agent_id").references(() => agents.id, { onDelete: "set null" }), + cause: text("cause").notNull(), + fingerprint: text("fingerprint").notNull(), + evidence: jsonb("evidence").$type>().notNull().default({}), + nextAction: text("next_action").notNull(), + wakePolicy: jsonb("wake_policy").$type>(), + monitorPolicy: jsonb("monitor_policy").$type>(), + attemptCount: integer("attempt_count").notNull().default(0), + maxAttempts: integer("max_attempts"), + timeoutAt: timestamp("timeout_at", { withTimezone: true }), + lastAttemptAt: timestamp("last_attempt_at", { withTimezone: true }), + outcome: text("outcome"), + resolutionNote: text("resolution_note"), + resolvedAt: timestamp("resolved_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => ({ + companySourceStatusIdx: index("issue_recovery_actions_company_source_status_idx").on( + table.companyId, + table.sourceIssueId, + table.status, + ), + companyOwnerStatusIdx: index("issue_recovery_actions_company_owner_status_idx").on( + table.companyId, + table.ownerAgentId, + table.status, + ), + companyRecoveryIssueIdx: index("issue_recovery_actions_company_recovery_issue_idx").on( + table.companyId, + table.recoveryIssueId, + ), + activeSourceIdx: uniqueIndex("issue_recovery_actions_active_source_uq") + .on(table.companyId, table.sourceIssueId) + .where(sql`${table.status} in ('active', 'escalated')`), + activeFingerprintIdx: uniqueIndex("issue_recovery_actions_active_fingerprint_uq") + .on(table.companyId, table.sourceIssueId, table.cause, table.fingerprint) + .where(sql`${table.status} in ('active', 'escalated')`), + }), +); diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 640f7563c05..ca730ff9736 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -214,6 +214,40 @@ export type IssueOriginKind = BuiltInIssueOriginKind | PluginIssueOriginKind; export const ISSUE_SURFACE_VISIBILITIES = ["default", "plugin_operation"] as const; export type IssueSurfaceVisibility = (typeof ISSUE_SURFACE_VISIBILITIES)[number]; +export const ISSUE_RECOVERY_ACTION_KINDS = [ + "missing_disposition", + "stranded_assigned_issue", + "active_run_watchdog", + "issue_graph_liveness", +] as const; +export type IssueRecoveryActionKind = (typeof ISSUE_RECOVERY_ACTION_KINDS)[number]; + +export const ISSUE_RECOVERY_ACTION_STATUSES = [ + "active", + "escalated", + "resolved", + "cancelled", +] as const; +export type IssueRecoveryActionStatus = (typeof ISSUE_RECOVERY_ACTION_STATUSES)[number]; + +export const ISSUE_RECOVERY_ACTION_OWNER_TYPES = [ + "agent", + "user", + "board", + "system", +] as const; +export type IssueRecoveryActionOwnerType = (typeof ISSUE_RECOVERY_ACTION_OWNER_TYPES)[number]; + +export const ISSUE_RECOVERY_ACTION_OUTCOMES = [ + "restored", + "delegated", + "false_positive", + "blocked", + "escalated", + "cancelled", +] as const; +export type IssueRecoveryActionOutcome = (typeof ISSUE_RECOVERY_ACTION_OUTCOMES)[number]; + export function pluginOperationIssueOriginKind(pluginKey: string): PluginIssueOriginKind { return `plugin:${pluginKey}:operation`; } diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 2239bbf4cb7..f4699968515 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -31,6 +31,10 @@ export { ISSUE_THREAD_INTERACTION_CONTINUATION_POLICIES, ISSUE_ORIGIN_KINDS, ISSUE_SURFACE_VISIBILITIES, + ISSUE_RECOVERY_ACTION_KINDS, + ISSUE_RECOVERY_ACTION_STATUSES, + ISSUE_RECOVERY_ACTION_OWNER_TYPES, + ISSUE_RECOVERY_ACTION_OUTCOMES, pluginOperationIssueOriginKind, isPluginOperationIssueOriginKind, ISSUE_RELATION_TYPES, @@ -149,6 +153,10 @@ export { type PluginIssueOriginKind, type IssueOriginKind, type IssueSurfaceVisibility, + type IssueRecoveryActionKind, + type IssueRecoveryActionStatus, + type IssueRecoveryActionOwnerType, + type IssueRecoveryActionOutcome, type IssueRelationType, type IssueTreeControlMode, type IssueTreeHoldReleasePolicyStrategy, @@ -373,6 +381,7 @@ export type { IssueBlockerAttentionState, IssueProductivityReview, IssueProductivityReviewTrigger, + IssueRecoveryAction, SuccessfulRunHandoffState, SuccessfulRunHandoffStateKind, IssueScheduledRetry, diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index 39ad1993c95..9f820d4590d 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -151,6 +151,7 @@ export type { IssueBlockerAttentionState, IssueProductivityReview, IssueProductivityReviewTrigger, + IssueRecoveryAction, SuccessfulRunHandoffState, SuccessfulRunHandoffStateKind, IssueScheduledRetry, diff --git a/packages/shared/src/types/issue.ts b/packages/shared/src/types/issue.ts index 827c259f029..495eee829d3 100644 --- a/packages/shared/src/types/issue.ts +++ b/packages/shared/src/types/issue.ts @@ -15,6 +15,10 @@ import type { IssueExecutionStateStatus, IssueOriginKind, IssuePriority, + IssueRecoveryActionKind, + IssueRecoveryActionOutcome, + IssueRecoveryActionOwnerType, + IssueRecoveryActionStatus, IssueWorkMode, ModelProfileKey, IssueThreadInteractionContinuationPolicy, @@ -169,6 +173,35 @@ export interface IssueProductivityReview { updatedAt: Date; } +export interface IssueRecoveryAction { + id: string; + companyId: string; + sourceIssueId: string; + recoveryIssueId: string | null; + kind: IssueRecoveryActionKind; + status: IssueRecoveryActionStatus; + ownerType: IssueRecoveryActionOwnerType; + ownerAgentId: string | null; + ownerUserId: string | null; + previousOwnerAgentId: string | null; + returnOwnerAgentId: string | null; + cause: string; + fingerprint: string; + evidence: Record; + nextAction: string; + wakePolicy: Record | null; + monitorPolicy: Record | null; + attemptCount: number; + maxAttempts: number | null; + timeoutAt: Date | string | null; + lastAttemptAt: Date | string | null; + outcome: IssueRecoveryActionOutcome | null; + resolutionNote: string | null; + resolvedAt: Date | string | null; + createdAt: Date | string; + updatedAt: Date | string; +} + export type SuccessfulRunHandoffStateKind = "required" | "resolved" | "escalated"; export interface SuccessfulRunHandoffState { @@ -372,6 +405,7 @@ export interface Issue { blocks?: IssueRelationIssueSummary[]; blockerAttention?: IssueBlockerAttention; productivityReview?: IssueProductivityReview | null; + activeRecoveryAction?: IssueRecoveryAction | null; successfulRunHandoff?: SuccessfulRunHandoffState | null; scheduledRetry?: IssueScheduledRetry | null; relatedWork?: IssueRelatedWorkSummary; diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.ts index 14b30989d17..3f9b40fd62c 100644 --- a/packages/shared/src/validators/index.ts +++ b/packages/shared/src/validators/index.ts @@ -156,6 +156,7 @@ export { updateIssueSchema, issueExecutionPolicySchema, issueExecutionStateSchema, + issueRecoveryActionReadModelSchema, issueReviewRequestSchema, issueExecutionWorkspaceSettingsSchema, checkoutIssueSchema, @@ -198,6 +199,7 @@ export { type CreateIssueLabel, type UpdateIssue, type IssueExecutionWorkspaceSettings, + type IssueRecoveryActionReadModel, type CheckoutIssue, type AddIssueComment, type CreateIssueThreadInteraction, diff --git a/packages/shared/src/validators/issue.ts b/packages/shared/src/validators/issue.ts index 0f73f4d9d7f..4ab3206f4b6 100644 --- a/packages/shared/src/validators/issue.ts +++ b/packages/shared/src/validators/issue.ts @@ -14,6 +14,10 @@ import { ISSUE_COMMENT_PRESENTATION_TONES, ISSUE_MONITOR_SCHEDULED_BY, ISSUE_PRIORITIES, + ISSUE_RECOVERY_ACTION_KINDS, + ISSUE_RECOVERY_ACTION_OUTCOMES, + ISSUE_RECOVERY_ACTION_OWNER_TYPES, + ISSUE_RECOVERY_ACTION_STATUSES, ISSUE_WORK_MODES, clampIssueRequestDepth, ISSUE_STATUSES, @@ -167,6 +171,37 @@ export const issueExecutionStateSchema = z.object({ monitor: issueExecutionMonitorStateSchema.optional().nullable(), }); +export const issueRecoveryActionReadModelSchema = z.object({ + id: z.string().uuid(), + companyId: z.string().uuid(), + sourceIssueId: z.string().uuid(), + recoveryIssueId: z.string().uuid().nullable(), + kind: z.enum(ISSUE_RECOVERY_ACTION_KINDS), + status: z.enum(ISSUE_RECOVERY_ACTION_STATUSES), + ownerType: z.enum(ISSUE_RECOVERY_ACTION_OWNER_TYPES), + ownerAgentId: z.string().uuid().nullable(), + ownerUserId: z.string().nullable(), + previousOwnerAgentId: z.string().uuid().nullable(), + returnOwnerAgentId: z.string().uuid().nullable(), + cause: z.string().min(1), + fingerprint: z.string().min(1), + evidence: z.record(z.unknown()), + nextAction: z.string().min(1), + wakePolicy: z.record(z.unknown()).nullable(), + monitorPolicy: z.record(z.unknown()).nullable(), + attemptCount: z.number().int().nonnegative(), + maxAttempts: z.number().int().positive().nullable(), + timeoutAt: z.union([z.date(), z.string().datetime()]).nullable(), + lastAttemptAt: z.union([z.date(), z.string().datetime()]).nullable(), + outcome: z.enum(ISSUE_RECOVERY_ACTION_OUTCOMES).nullable(), + resolutionNote: z.string().nullable(), + resolvedAt: z.union([z.date(), z.string().datetime()]).nullable(), + createdAt: z.union([z.date(), z.string().datetime()]), + updatedAt: z.union([z.date(), z.string().datetime()]), +}); + +export type IssueRecoveryActionReadModel = z.infer; + const issueRequestDepthInputSchema = z .number() .int() diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts new file mode 100644 index 00000000000..68d81d189c0 --- /dev/null +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -0,0 +1,280 @@ +import { randomUUID } from "node:crypto"; +import express from "express"; +import request from "supertest"; +import { and, eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, + activityLog, + companies, + createDb, + issueComments, + issueRecoveryActions, + issues, +} from "@paperclipai/db"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; +import { issueRoutes } from "../routes/issues.js"; +import { issueRecoveryActionService } from "../services/issue-recovery-actions.js"; +import { recoveryService } from "../services/recovery/service.js"; + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping embedded Postgres issue recovery action tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +describeEmbeddedPostgres("issue recovery actions", () => { + let tempDb: Awaited> | null = null; + let db: ReturnType; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-recovery-actions-"); + db = createDb(tempDb.connectionString); + }, 30_000); + + afterEach(async () => { + await db.delete(issueRecoveryActions); + await db.delete(issueComments); + await db.delete(activityLog); + await db.delete(issues); + await db.delete(agents); + await db.delete(companies); + }); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + async function seedCompany() { + const companyId = randomUUID(); + const managerId = randomUUID(); + const coderId = randomUUID(); + const sourceIssueId = randomUUID(); + const prefix = `RA${companyId.replaceAll("-", "").slice(0, 6).toUpperCase()}`; + await db.insert(companies).values({ + id: companyId, + name: "Recovery Co", + issuePrefix: prefix, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(agents).values([ + { + id: managerId, + companyId, + name: "CTO", + role: "cto", + status: "idle", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }, + { + id: coderId, + companyId, + name: "Coder", + role: "engineer", + status: "idle", + reportsTo: managerId, + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }, + ]); + await db.insert(issues).values({ + id: sourceIssueId, + companyId, + title: "Implement backend recovery", + status: "in_progress", + priority: "medium", + assigneeAgentId: coderId, + issueNumber: 1, + identifier: `${prefix}-1`, + }); + const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId)); + return { companyId, managerId, coderId, sourceIssueId, prefix, sourceIssue: sourceIssue! }; + } + + it("upserts one active source-scoped action per issue and keeps company scoping explicit", async () => { + const { companyId, managerId, sourceIssueId } = await seedCompany(); + const svc = issueRecoveryActionService(db); + + const first = await svc.upsertSourceScoped({ + companyId, + sourceIssueId, + kind: "stranded_assigned_issue", + ownerType: "agent", + ownerAgentId: managerId, + cause: "stranded_assigned_issue", + fingerprint: "recovery:fingerprint", + evidence: { latestRunId: "run-1" }, + nextAction: "Restore a live execution path.", + wakePolicy: { type: "wake_owner" }, + }); + const second = await svc.upsertSourceScoped({ + companyId, + sourceIssueId, + kind: "stranded_assigned_issue", + ownerType: "agent", + ownerAgentId: managerId, + cause: "stranded_assigned_issue", + fingerprint: "recovery:fingerprint", + evidence: { latestRunId: "run-2" }, + nextAction: "Restore a live execution path.", + wakePolicy: { type: "wake_owner" }, + }); + + expect(second.id).toBe(first.id); + expect(second.attemptCount).toBe(2); + expect(second.evidence).toMatchObject({ latestRunId: "run-2" }); + expect(await svc.getActiveForIssue(companyId, sourceIssueId)).toMatchObject({ id: first.id }); + expect(await svc.getActiveForIssue(randomUUID(), sourceIssueId)).toBeNull(); + }); + + it("escalates stranded assigned work into a source action instead of a recovery issue", async () => { + const { companyId, managerId, coderId, sourceIssue } = await seedCompany(); + const enqueueWakeup = vi.fn(async () => null); + const recovery = recoveryService(db, { enqueueWakeup }); + const latestRun = { + id: randomUUID(), + agentId: coderId, + status: "failed", + error: "adapter failed", + errorCode: "adapter_failed", + contextSnapshot: { retryReason: "issue_continuation_needed" }, + livenessState: "needs_followup", + } as const; + + await recovery.escalateStrandedAssignedIssue({ + issue: sourceIssue, + previousStatus: "in_progress", + latestRun, + comment: "Automatic continuation recovery failed.", + }); + await recovery.escalateStrandedAssignedIssue({ + issue: sourceIssue, + previousStatus: "in_progress", + latestRun, + comment: "Automatic continuation recovery failed.", + }); + + const actionRows = await db + .select() + .from(issueRecoveryActions) + .where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id)); + expect(actionRows).toHaveLength(1); + expect(actionRows[0]).toMatchObject({ + companyId, + kind: "stranded_assigned_issue", + status: "active", + ownerAgentId: managerId, + previousOwnerAgentId: coderId, + returnOwnerAgentId: coderId, + cause: "stranded_assigned_issue", + attemptCount: 2, + }); + + const [updatedIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id)); + expect(updatedIssue).toMatchObject({ + status: "blocked", + assigneeAgentId: managerId, + }); + const recoveryIssues = await db + .select() + .from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery"))); + expect(recoveryIssues).toHaveLength(0); + expect(enqueueWakeup).toHaveBeenCalledTimes(2); + expect(enqueueWakeup.mock.calls[0]?.[1]?.payload).toMatchObject({ + issueId: sourceIssue.id, + sourceIssueId: sourceIssue.id, + recoveryCause: "stranded_assigned_issue", + }); + }); + + it("does not create nested recovery artifacts when issue-backed fallback work itself fails", async () => { + const { companyId, managerId, sourceIssueId, prefix } = await seedCompany(); + const recoveryIssueId = randomUUID(); + await db.insert(issues).values({ + id: recoveryIssueId, + companyId, + title: "Recover stalled issue", + status: "in_progress", + priority: "medium", + assigneeAgentId: managerId, + parentId: sourceIssueId, + issueNumber: 2, + identifier: `${prefix}-2`, + originKind: "stranded_issue_recovery", + originId: sourceIssueId, + originFingerprint: `stranded_issue_recovery:${sourceIssueId}`, + }); + const [recoveryIssue] = await db.select().from(issues).where(eq(issues.id, recoveryIssueId)); + const recovery = recoveryService(db, { enqueueWakeup: vi.fn(async () => null) }); + + await recovery.escalateStrandedAssignedIssue({ + issue: recoveryIssue!, + previousStatus: "in_progress", + latestRun: { + id: randomUUID(), + agentId: managerId, + status: "failed", + error: "adapter failed", + errorCode: "adapter_failed", + contextSnapshot: { retryReason: "issue_continuation_needed" }, + livenessState: "needs_followup", + }, + }); + + const actionRows = await db.select().from(issueRecoveryActions); + expect(actionRows).toHaveLength(0); + const recoveryIssues = await db + .select() + .from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery"))); + expect(recoveryIssues).toHaveLength(1); + expect(recoveryIssues[0]?.status).toBe("blocked"); + }); + + it("exposes active recovery actions on the issue read API", async () => { + const { companyId, managerId, sourceIssueId } = await seedCompany(); + const recoveryActionSvc = issueRecoveryActionService(db); + const action = await recoveryActionSvc.upsertSourceScoped({ + companyId, + sourceIssueId, + kind: "missing_disposition", + ownerType: "agent", + ownerAgentId: managerId, + cause: "successful_run_missing_issue_disposition", + fingerprint: "missing-disposition:fingerprint", + evidence: { sourceRunId: "run-1" }, + nextAction: "Choose a valid issue disposition.", + wakePolicy: { type: "wake_owner" }, + }); + const app = express(); + app.use((req, _res, next) => { + (req as any).actor = { type: "board", source: "local_implicit" }; + next(); + }); + app.use("/api", issueRoutes(db, {} as any)); + + const detail = await request(app).get(`/api/issues/${sourceIssueId}`).expect(200); + expect(detail.body.activeRecoveryAction).toMatchObject({ + id: action.id, + sourceIssueId, + kind: "missing_disposition", + ownerAgentId: managerId, + }); + + const list = await request(app).get(`/api/issues/${sourceIssueId}/recovery-actions`).expect(200); + expect(list.body.active).toMatchObject({ id: action.id }); + expect(list.body.actions).toHaveLength(1); + }); +}); diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 1b7a5a7d16c..a6245fb4768 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -53,6 +53,7 @@ import { goalService, heartbeatService, issueApprovalService, + issueRecoveryActionService, issueThreadInteractionService, ISSUE_LIST_DEFAULT_LIMIT, ISSUE_LIST_MAX_LIMIT, @@ -787,6 +788,7 @@ export function issueRoutes( const projectsSvc = projectService(db); const goalsSvc = goalService(db); const issueApprovalsSvc = issueApprovalService(db); + const recoveryActionsSvc = issueRecoveryActionService(db); const executionWorkspacesSvc = executionWorkspaceServiceDirect(db); const workProductsSvc = workProductService(db); const documentsSvc = documentService(db); @@ -1541,6 +1543,7 @@ export function issueRoutes( attachments, continuationSummary, currentExecutionWorkspace, + activeRecoveryAction, ] = await Promise.all([ resolveIssueProjectAndGoal(issue), @@ -1554,6 +1557,7 @@ export function issueRoutes( svc.listAttachments(issue.id), documentsSvc.getIssueDocumentByKey(issue.id, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY), currentExecutionWorkspacePromise, + recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id), ]); res.json({ @@ -1567,6 +1571,7 @@ export function issueRoutes( ...(blockerAttention ? { blockerAttention } : {}), productivityReview, scheduledRetry, + activeRecoveryAction, priority: issue.priority, projectId: issue.projectId, goalId: goal?.id ?? issue.goalId, @@ -1649,6 +1654,7 @@ export function issueRoutes( referenceSummary, successfulRunHandoffStates, scheduledRetry, + activeRecoveryAction, ] = await Promise.all([ resolveIssueProjectAndGoal(issue), svc.getAncestors(issue.id), @@ -1660,6 +1666,7 @@ export function issueRoutes( issueReferencesSvc.listIssueReferenceSummary(issue.id), listSuccessfulRunHandoffStates(db, issue.companyId, [issue.id]), svc.getCurrentScheduledRetry(issue.id), + recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id), ]); const mentionedProjects = mentionedProjectIds.length > 0 ? await projectsSvc.listByIds(issue.companyId, mentionedProjectIds) @@ -1676,6 +1683,7 @@ export function issueRoutes( productivityReview, successfulRunHandoff: successfulRunHandoffStates.get(issue.id) ?? null, scheduledRetry, + activeRecoveryAction, blockedBy: relations.blockedBy, blocks: relations.blocks, relatedWork: referenceSummary, @@ -1689,6 +1697,21 @@ export function issueRoutes( }); }); + router.get("/issues/:id/recovery-actions", async (req, res) => { + const id = req.params.id as string; + const issue = await svc.getById(id); + if (!issue) { + res.status(404).json({ error: "Issue not found" }); + return; + } + assertCompanyAccess(req, issue.companyId); + const active = await recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id); + res.json({ + active, + actions: active ? [active] : [], + }); + }); + router.get("/issues/:id/work-products", async (req, res) => { const id = req.params.id as string; const issue = await svc.getById(id); diff --git a/server/src/services/index.ts b/server/src/services/index.ts index 5cdf0a1d9c1..2e7c6bcdaf5 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -24,6 +24,7 @@ export { issueThreadInteractionService } from "./issue-thread-interactions.js"; export { issueTreeControlService } from "./issue-tree-control.js"; export { issueApprovalService } from "./issue-approvals.js"; export { issueReferenceService } from "./issue-references.js"; +export { issueRecoveryActionService } from "./issue-recovery-actions.js"; export { goalService } from "./goals.js"; export { activityService, type ActivityFilters } from "./activity.js"; export { approvalService } from "./approvals.js"; diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts new file mode 100644 index 00000000000..c34706c8eb3 --- /dev/null +++ b/server/src/services/issue-recovery-actions.ts @@ -0,0 +1,197 @@ +import { and, desc, eq, inArray } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { issueRecoveryActions } from "@paperclipai/db"; +import type { + IssueRecoveryAction, + IssueRecoveryActionKind, + IssueRecoveryActionOwnerType, + IssueRecoveryActionStatus, +} from "@paperclipai/shared"; + +const ACTIVE_RECOVERY_ACTION_STATUSES = ["active", "escalated"] as const satisfies readonly IssueRecoveryActionStatus[]; + +type IssueRecoveryActionRow = typeof issueRecoveryActions.$inferSelect; + +export type UpsertIssueRecoveryActionInput = { + companyId: string; + sourceIssueId: string; + recoveryIssueId?: string | null; + kind: IssueRecoveryActionKind; + ownerType?: IssueRecoveryActionOwnerType; + ownerAgentId?: string | null; + ownerUserId?: string | null; + previousOwnerAgentId?: string | null; + returnOwnerAgentId?: string | null; + cause: string; + fingerprint: string; + evidence?: Record; + nextAction: string; + wakePolicy?: Record | null; + monitorPolicy?: Record | null; + maxAttempts?: number | null; + timeoutAt?: Date | null; + lastAttemptAt?: Date | null; +}; + +function toReadModel(row: IssueRecoveryActionRow): IssueRecoveryAction { + return { + id: row.id, + companyId: row.companyId, + sourceIssueId: row.sourceIssueId, + recoveryIssueId: row.recoveryIssueId, + kind: row.kind as IssueRecoveryAction["kind"], + status: row.status as IssueRecoveryAction["status"], + ownerType: row.ownerType as IssueRecoveryAction["ownerType"], + ownerAgentId: row.ownerAgentId, + ownerUserId: row.ownerUserId, + previousOwnerAgentId: row.previousOwnerAgentId, + returnOwnerAgentId: row.returnOwnerAgentId, + cause: row.cause, + fingerprint: row.fingerprint, + evidence: row.evidence, + nextAction: row.nextAction, + wakePolicy: row.wakePolicy, + monitorPolicy: row.monitorPolicy, + attemptCount: row.attemptCount, + maxAttempts: row.maxAttempts, + timeoutAt: row.timeoutAt, + lastAttemptAt: row.lastAttemptAt, + outcome: row.outcome as IssueRecoveryAction["outcome"], + resolutionNote: row.resolutionNote, + resolvedAt: row.resolvedAt, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} + +function isUniqueRecoveryActionConflict(error: unknown) { + const maybe = error as { code?: string; constraint?: string; message?: string } | null; + return Boolean( + maybe && + maybe.code === "23505" && + ( + maybe.constraint === "issue_recovery_actions_active_source_uq" || + maybe.constraint === "issue_recovery_actions_active_fingerprint_uq" || + typeof maybe.message === "string" && ( + maybe.message.includes("issue_recovery_actions_active_source_uq") || + maybe.message.includes("issue_recovery_actions_active_fingerprint_uq") + ) + ), + ); +} + +export function issueRecoveryActionService(db: Db) { + async function getActiveForIssue(companyId: string, sourceIssueId: string): Promise { + const row = await db + .select() + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + eq(issueRecoveryActions.sourceIssueId, sourceIssueId), + inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), + ), + ) + .orderBy(desc(issueRecoveryActions.updatedAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + return row ? toReadModel(row) : null; + } + + async function listActiveForIssues(companyId: string, sourceIssueIds: string[]) { + if (sourceIssueIds.length === 0) return new Map(); + const rows = await db + .select() + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + inArray(issueRecoveryActions.sourceIssueId, [...new Set(sourceIssueIds)]), + inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), + ), + ) + .orderBy(desc(issueRecoveryActions.updatedAt)); + const result = new Map(); + for (const row of rows) { + if (!result.has(row.sourceIssueId)) result.set(row.sourceIssueId, toReadModel(row)); + } + return result; + } + + async function upsertSourceScoped(input: UpsertIssueRecoveryActionInput): Promise { + const existing = await getActiveForIssue(input.companyId, input.sourceIssueId); + const now = new Date(); + const ownerType = input.ownerType ?? (input.ownerAgentId ? "agent" : "board"); + if (existing) { + const [updated] = await db + .update(issueRecoveryActions) + .set({ + recoveryIssueId: input.recoveryIssueId ?? null, + kind: input.kind, + status: "active", + ownerType, + ownerAgentId: input.ownerAgentId ?? null, + ownerUserId: input.ownerUserId ?? null, + previousOwnerAgentId: input.previousOwnerAgentId ?? existing.previousOwnerAgentId, + returnOwnerAgentId: input.returnOwnerAgentId ?? existing.returnOwnerAgentId, + cause: input.cause, + fingerprint: input.fingerprint, + evidence: input.evidence ?? existing.evidence, + nextAction: input.nextAction, + wakePolicy: input.wakePolicy ?? null, + monitorPolicy: input.monitorPolicy ?? null, + attemptCount: existing.attemptCount + 1, + maxAttempts: input.maxAttempts ?? null, + timeoutAt: input.timeoutAt ?? null, + lastAttemptAt: input.lastAttemptAt ?? now, + outcome: null, + resolutionNote: null, + resolvedAt: null, + updatedAt: now, + }) + .where(eq(issueRecoveryActions.id, existing.id)) + .returning(); + return toReadModel(updated!); + } + + try { + const [created] = await db + .insert(issueRecoveryActions) + .values({ + companyId: input.companyId, + sourceIssueId: input.sourceIssueId, + recoveryIssueId: input.recoveryIssueId ?? null, + kind: input.kind, + status: "active", + ownerType, + ownerAgentId: input.ownerAgentId ?? null, + ownerUserId: input.ownerUserId ?? null, + previousOwnerAgentId: input.previousOwnerAgentId ?? null, + returnOwnerAgentId: input.returnOwnerAgentId ?? null, + cause: input.cause, + fingerprint: input.fingerprint, + evidence: input.evidence ?? {}, + nextAction: input.nextAction, + wakePolicy: input.wakePolicy ?? null, + monitorPolicy: input.monitorPolicy ?? null, + attemptCount: 1, + maxAttempts: input.maxAttempts ?? null, + timeoutAt: input.timeoutAt ?? null, + lastAttemptAt: input.lastAttemptAt ?? now, + }) + .returning(); + return toReadModel(created!); + } catch (error) { + if (!isUniqueRecoveryActionConflict(error)) throw error; + const raced = await getActiveForIssue(input.companyId, input.sourceIssueId); + if (!raced) throw error; + return upsertSourceScoped(input); + } + } + + return { + getActiveForIssue, + listActiveForIssues, + upsertSourceScoped, + }; +} diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 2d4ba3ffc74..c45c2eaf7e9 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -17,6 +17,7 @@ import { issueAttachments, issueInboxArchives, issueLabels, + issueRecoveryActions, issueRelations, issueComments, issueDocuments, @@ -1270,6 +1271,18 @@ async function listIssueBlockerAttentionMap( explicitWaitingIssueIds.add(parsed.issueId); explicitWaitingIssueIds.add(parsed.leafIssueId); } + + const recoveryActionRows: Array<{ sourceIssueId: string }> = await dbOrTx + .select({ sourceIssueId: issueRecoveryActions.sourceIssueId }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + inArray(issueRecoveryActions.status, ["active", "escalated"]), + inArray(issueRecoveryActions.sourceIssueId, explicitWaitCandidateIds), + ), + ); + for (const row of recoveryActionRows) explicitWaitingIssueIds.add(row.sourceIssueId); } const agentRows: IssueBlockerAttentionAgentRow[] = agentIds.size > 0 diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 3aee7d2b3f1..26cd613ffab 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -16,6 +16,7 @@ import { heartbeatRunWatchdogDecisions, heartbeatRuns, issueApprovals, + issueRecoveryActions, issueRelations, issueThreadInteractions, issues, @@ -29,6 +30,7 @@ import { redactSensitiveText } from "../../redaction.js"; import { logActivity } from "../activity-log.js"; import { budgetService } from "../budgets.js"; import { instanceSettingsService } from "../instance-settings.js"; +import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { issueTreeControlService } from "../issue-tree-control.js"; import { issueService } from "../issues.js"; import { getRunLogStore } from "../run-log-store.js"; @@ -356,6 +358,7 @@ function buildLivenessOriginalIssueComment(finding: IssueLivenessFinding, escala export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) { const issuesSvc = issueService(db); + const recoveryActionsSvc = issueRecoveryActionService(db); const treeControlSvc = issueTreeControlService(db); const budgets = budgetService(db); const instanceSettings = instanceSettingsService(db); @@ -1534,6 +1537,135 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) return recovery; } + function strandedRecoveryActionKind(cause: StrandedRecoveryCause) { + return cause === SUCCESSFUL_RUN_MISSING_STATE_REASON + ? "missing_disposition" as const + : "stranded_assigned_issue" as const; + } + + function strandedRecoveryActionFingerprint(input: { + issue: typeof issues.$inferSelect; + recoveryCause: StrandedRecoveryCause; + latestRun: LatestIssueRun; + successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null; + }) { + const sourceRunId = input.successfulRunHandoffEvidence?.sourceRunId ?? input.latestRun?.id ?? "no-run"; + return [ + "source_scoped_recovery", + input.issue.companyId, + input.issue.id, + input.recoveryCause, + sourceRunId, + ].join(":"); + } + + function buildStrandedRecoveryActionEvidence(input: { + issue: typeof issues.$inferSelect; + latestRun: LatestIssueRun; + previousStatus: "todo" | "in_progress"; + recoveryCause: StrandedRecoveryCause; + successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null; + }) { + const context = parseObject(input.latestRun?.contextSnapshot); + return { + sourceIssueId: input.issue.id, + sourceIdentifier: input.issue.identifier, + previousStatus: input.previousStatus, + latestIssueStatus: input.issue.status, + latestRunId: input.latestRun?.id ?? null, + latestRunStatus: input.latestRun?.status ?? null, + latestRunErrorCode: input.latestRun?.errorCode ?? null, + retryReason: readNonEmptyString(context.retryReason) ?? null, + recoveryCause: input.recoveryCause, + sourceRunId: input.successfulRunHandoffEvidence?.sourceRunId ?? null, + correctiveRunId: input.successfulRunHandoffEvidence?.correctiveRunId ?? null, + missingDisposition: input.successfulRunHandoffEvidence?.missingDisposition ?? null, + handoffAttempt: input.successfulRunHandoffEvidence?.handoffAttempt ?? null, + maxHandoffAttempts: input.successfulRunHandoffEvidence?.maxHandoffAttempts ?? null, + }; + } + + async function ensureSourceScopedStrandedRecoveryAction(input: { + issue: typeof issues.$inferSelect; + latestRun: LatestIssueRun; + previousStatus: "todo" | "in_progress"; + recoveryCause?: StrandedRecoveryCause; + successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null; + }) { + const recoveryCause = input.recoveryCause ?? "stranded_assigned_issue"; + const ownerAgentId = await resolveStrandedIssueRecoveryOwnerAgentId(input.issue); + const now = new Date(); + const action = await recoveryActionsSvc.upsertSourceScoped({ + companyId: input.issue.companyId, + sourceIssueId: input.issue.id, + kind: strandedRecoveryActionKind(recoveryCause), + ownerType: ownerAgentId ? "agent" : "board", + ownerAgentId, + previousOwnerAgentId: input.issue.assigneeAgentId, + returnOwnerAgentId: input.issue.assigneeAgentId, + cause: recoveryCause, + fingerprint: strandedRecoveryActionFingerprint({ + issue: input.issue, + recoveryCause, + latestRun: input.latestRun, + successfulRunHandoffEvidence: input.successfulRunHandoffEvidence, + }), + evidence: buildStrandedRecoveryActionEvidence({ + issue: input.issue, + latestRun: input.latestRun, + previousStatus: input.previousStatus, + recoveryCause, + successfulRunHandoffEvidence: input.successfulRunHandoffEvidence, + }), + nextAction: recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON + ? "Choose and record a valid issue disposition without copying transcript content." + : "Restore a live execution path, fix the runtime/adapter failure, or record an intentional manual resolution.", + wakePolicy: ownerAgentId + ? { + type: "wake_owner", + reason: "source_scoped_recovery_action", + ownerAgentId, + } + : { + type: "board_escalation", + reason: "no_invokable_recovery_owner", + }, + monitorPolicy: null, + maxAttempts: 1, + lastAttemptAt: now, + }); + + if (ownerAgentId) { + await deps.enqueueWakeup(ownerAgentId, { + source: "assignment", + triggerDetail: "system", + reason: "source_scoped_recovery_action", + idempotencyKey: `source_scoped_recovery_action:${action.id}:${action.attemptCount}`, + payload: withRecoveryModelProfileHint({ + issueId: input.issue.id, + sourceIssueId: input.issue.id, + recoveryActionId: action.id, + strandedRunId: input.latestRun?.id ?? null, + recoveryCause, + }), + requestedByActorType: "system", + requestedByActorId: null, + contextSnapshot: withRecoveryModelProfileHint({ + issueId: input.issue.id, + taskId: input.issue.id, + wakeReason: "source_scoped_recovery_action", + source: "issue_recovery_action", + recoveryActionId: action.id, + sourceIssueId: input.issue.id, + strandedRunId: input.latestRun?.id ?? null, + recoveryCause, + }), + }); + } + + return action; + } + function buildRecoveryIssueInPlaceEscalationComment(input: { issue: typeof issues.$inferSelect; previousStatus: "todo" | "in_progress"; @@ -1650,29 +1782,31 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) recoveryCause?: StrandedRecoveryCause; successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null; }) { - const nestedRecoverySuppressed = isStrandedIssueRecoveryIssue(input.issue); - let recoveryIssue: typeof issues.$inferSelect | null = null; - if (!nestedRecoverySuppressed) { - recoveryIssue = await ensureStrandedIssueRecoveryIssue({ + if (isStrandedIssueRecoveryIssue(input.issue)) { + return escalateStrandedRecoveryIssueInPlace({ issue: input.issue, previousStatus: input.previousStatus, latestRun: input.latestRun, - recoveryCause: input.recoveryCause, - successfulRunHandoffEvidence: input.successfulRunHandoffEvidence, }); } + + const recoveryAction = await ensureSourceScopedStrandedRecoveryAction({ + issue: input.issue, + previousStatus: input.previousStatus, + latestRun: input.latestRun, + recoveryCause: input.recoveryCause, + successfulRunHandoffEvidence: input.successfulRunHandoffEvidence, + }); const blockerIds = await existingUnresolvedBlockerIssueIds(input.issue.companyId, input.issue.id); - const nextBlockerIds = recoveryIssue - ? [...new Set([...blockerIds, recoveryIssue.id])] - : blockerIds; const updated = await issuesSvc.update(input.issue.id, { status: "blocked", - blockedByIssueIds: nextBlockerIds, + blockedByIssueIds: blockerIds, + assigneeAgentId: recoveryAction.ownerAgentId ?? input.issue.assigneeAgentId, }); if (!updated) return null; const prefix = await getCompanyIssuePrefix(input.issue.companyId); - const recoveryOwner = recoveryIssue?.assigneeAgentId ? await getAgent(recoveryIssue.assigneeAgentId) : null; + const recoveryOwner = recoveryAction.ownerAgentId ? await getAgent(recoveryAction.ownerAgentId) : null; const sourceAssignee = input.issue.assigneeAgentId ? await getAgent(input.issue.assigneeAgentId) : null; let notice: SuccessfulRunHandoffNotice | null = null; if (input.recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON && input.successfulRunHandoffEvidence) { @@ -1683,30 +1817,26 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) : null, correctiveRun: input.latestRun ? { id: input.latestRun.id, status: input.latestRun.status } : null, sourceAssignee, - recoveryIssue, + recoveryIssue: null, recoveryOwner, latestIssueStatus: input.issue.status, latestHandoffRunStatus: input.latestRun?.status ?? "unknown", missingDisposition: input.successfulRunHandoffEvidence.missingDisposition, }); } - let recoveryLine: string; - if (nestedRecoverySuppressed) { - recoveryLine = await buildNestedStrandedRecoveryLine(input.issue, prefix); - } else if (recoveryIssue) { - recoveryLine = [ + const recoveryLine = recoveryAction.ownerAgentId + ? [ "", - `- Recovery issue: ${issueUiLink({ identifier: recoveryIssue.identifier, id: recoveryIssue.id }, prefix)}`, + `- Recovery action: \`${recoveryAction.id}\``, `- Recovery owner: ${agentUiLink(recoveryOwner, prefix)}`, - "- Next action: the recovery owner should either restore a live execution path or record the manual resolution, then mark the recovery issue done.", - ].join("\n"); - } else { - recoveryLine = [ + "- Next action: the recovery owner should either restore a live execution path or record the manual resolution on the source issue.", + ].join("\n") + : [ "", - "- Recovery issue: none created because Paperclip could not find an invokable manager, creator, or executive owner with budget available.", + `- Recovery action: \`${recoveryAction.id}\``, + "- Recovery owner: board escalation, because Paperclip could not find an invokable manager, creator, or executive owner with budget available.", "- Next action: a board operator should assign an invokable recovery owner, fix the agent/runtime state, or record an intentional manual resolution.", ].join("\n"); - } if (notice) { await issuesSvc.addComment(input.issue.id, notice.body, {}, { @@ -1740,9 +1870,11 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) latestRunId: input.latestRun?.id ?? null, latestRunStatus: input.latestRun?.status ?? null, latestRunErrorCode: input.latestRun?.errorCode ?? null, - recoveryIssueId: recoveryIssue?.id ?? null, - nestedRecoverySuppressed, - blockerIssueIds: nextBlockerIds, + recoveryActionId: recoveryAction.id, + recoveryOwnerAgentId: recoveryAction.ownerAgentId, + previousOwnerAgentId: recoveryAction.previousOwnerAgentId, + returnOwnerAgentId: recoveryAction.returnOwnerAgentId, + blockerIssueIds: blockerIds, }, }); @@ -2016,6 +2148,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) interactionRows, approvalRows, recoveryIssueRows, + recoveryActionRows, ] = await Promise.all([ db .select({ @@ -2132,6 +2265,14 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) notInArray(issues.status, ["done", "cancelled"]), ), ), + db + .select({ + companyId: issueRecoveryActions.companyId, + issueId: issueRecoveryActions.sourceIssueId, + status: issueRecoveryActions.status, + }) + .from(issueRecoveryActions) + .where(inArray(issueRecoveryActions.status, ["active", "escalated"])), ]); const openRecoveryIssues = recoveryIssueRows.flatMap((row) => { @@ -2185,7 +2326,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) })), pendingInteractions: interactionRows, pendingApprovals: approvalRows, - openRecoveryIssues, + openRecoveryIssues: openRecoveryIssues.concat(recoveryActionRows), now: new Date(), }); }