mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
Implement source-scoped recovery actions
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
e3af7aa489
commit
d498f8a9b2
16 files changed
+938
-28
No files matched your search
@@ -0,0 +1,64 @@
|
||||
CREATE TABLE IF NOT EXISTS "issue_recovery_actions" (
|
||||
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
|
||||
"company_id" uuid NOT NULL,
|
||||
"source_issue_id" uuid NOT NULL,
|
||||
"recovery_issue_id" uuid,
|
||||
"kind" text NOT NULL,
|
||||
"status" text DEFAULT 'active' NOT NULL,
|
||||
"owner_type" text DEFAULT 'agent' NOT NULL,
|
||||
"owner_agent_id" uuid,
|
||||
"owner_user_id" text,
|
||||
"previous_owner_agent_id" uuid,
|
||||
"return_owner_agent_id" uuid,
|
||||
"cause" text NOT NULL,
|
||||
"fingerprint" text NOT NULL,
|
||||
"evidence" jsonb DEFAULT '{}'::jsonb NOT NULL,
|
||||
"next_action" text NOT NULL,
|
||||
"wake_policy" jsonb,
|
||||
"monitor_policy" jsonb,
|
||||
"attempt_count" integer DEFAULT 0 NOT NULL,
|
||||
"max_attempts" integer,
|
||||
"timeout_at" timestamp with time zone,
|
||||
"last_attempt_at" timestamp with time zone,
|
||||
"outcome" text,
|
||||
"resolution_note" text,
|
||||
"resolved_at" timestamp with time zone,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
|
||||
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
DO $$ BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'issue_recovery_actions_company_id_companies_id_fk') THEN
|
||||
ALTER TABLE "issue_recovery_actions" ADD CONSTRAINT "issue_recovery_actions_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE no action ON UPDATE no action;
|
||||
END IF;
|
||||
END $$;--> statement-breakpoint
|
||||
DO $$ BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'issue_recovery_actions_source_issue_id_issues_id_fk') THEN
|
||||
ALTER TABLE "issue_recovery_actions" ADD CONSTRAINT "issue_recovery_actions_source_issue_id_issues_id_fk" FOREIGN KEY ("source_issue_id") REFERENCES "public"."issues"("id") ON DELETE cascade ON UPDATE no action;
|
||||
END IF;
|
||||
END $$;--> statement-breakpoint
|
||||
DO $$ BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'issue_recovery_actions_recovery_issue_id_issues_id_fk') THEN
|
||||
ALTER TABLE "issue_recovery_actions" ADD CONSTRAINT "issue_recovery_actions_recovery_issue_id_issues_id_fk" FOREIGN KEY ("recovery_issue_id") REFERENCES "public"."issues"("id") ON DELETE set null ON UPDATE no action;
|
||||
END IF;
|
||||
END $$;--> statement-breakpoint
|
||||
DO $$ BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'issue_recovery_actions_owner_agent_id_agents_id_fk') THEN
|
||||
ALTER TABLE "issue_recovery_actions" ADD CONSTRAINT "issue_recovery_actions_owner_agent_id_agents_id_fk" FOREIGN KEY ("owner_agent_id") REFERENCES "public"."agents"("id") ON DELETE set null ON UPDATE no action;
|
||||
END IF;
|
||||
END $$;--> statement-breakpoint
|
||||
DO $$ BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'issue_recovery_actions_previous_owner_agent_id_agents_id_fk') THEN
|
||||
ALTER TABLE "issue_recovery_actions" ADD CONSTRAINT "issue_recovery_actions_previous_owner_agent_id_agents_id_fk" FOREIGN KEY ("previous_owner_agent_id") REFERENCES "public"."agents"("id") ON DELETE set null ON UPDATE no action;
|
||||
END IF;
|
||||
END $$;--> statement-breakpoint
|
||||
DO $$ BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'issue_recovery_actions_return_owner_agent_id_agents_id_fk') THEN
|
||||
ALTER TABLE "issue_recovery_actions" ADD CONSTRAINT "issue_recovery_actions_return_owner_agent_id_agents_id_fk" FOREIGN KEY ("return_owner_agent_id") REFERENCES "public"."agents"("id") ON DELETE set null ON UPDATE no action;
|
||||
END IF;
|
||||
END $$;--> statement-breakpoint
|
||||
CREATE INDEX IF NOT EXISTS "issue_recovery_actions_company_source_status_idx" ON "issue_recovery_actions" USING btree ("company_id","source_issue_id","status");--> statement-breakpoint
|
||||
CREATE INDEX IF NOT EXISTS "issue_recovery_actions_company_owner_status_idx" ON "issue_recovery_actions" USING btree ("company_id","owner_agent_id","status");--> statement-breakpoint
|
||||
CREATE INDEX IF NOT EXISTS "issue_recovery_actions_company_recovery_issue_idx" ON "issue_recovery_actions" USING btree ("company_id","recovery_issue_id");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "issue_recovery_actions_active_source_uq" ON "issue_recovery_actions" USING btree ("company_id","source_issue_id") WHERE "issue_recovery_actions"."status" in ('active', 'escalated');--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "issue_recovery_actions_active_fingerprint_uq" ON "issue_recovery_actions" USING btree ("company_id","source_issue_id","cause","fingerprint") WHERE "issue_recovery_actions"."status" in ('active', 'escalated');
|
||||
@@ -589,6 +589,13 @@
|
||||
"when": 1778074536410,
|
||||
"tag": "0083_company_secret_provider_configs",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 84,
|
||||
"version": "7",
|
||||
"when": 1778355326070,
|
||||
"tag": "0084_issue_recovery_actions",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -29,6 +29,7 @@ export { workspaceRuntimeServices } from "./workspace_runtime_services.js";
|
||||
export { projectGoals } from "./project_goals.js";
|
||||
export { goals } from "./goals.js";
|
||||
export { issues } from "./issues.js";
|
||||
export { issueRecoveryActions } from "./issue_recovery_actions.js";
|
||||
export { issueReferenceMentions } from "./issue_reference_mentions.js";
|
||||
export { issueRelations } from "./issue_relations.js";
|
||||
export { routines, routineRevisions, routineTriggers, routineRuns } from "./routines.js";
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import {
|
||||
index,
|
||||
integer,
|
||||
jsonb,
|
||||
pgTable,
|
||||
text,
|
||||
timestamp,
|
||||
uniqueIndex,
|
||||
uuid,
|
||||
} from "drizzle-orm/pg-core";
|
||||
import { agents } from "./agents.js";
|
||||
import { companies } from "./companies.js";
|
||||
import { issues } from "./issues.js";
|
||||
|
||||
export const issueRecoveryActions = pgTable(
|
||||
"issue_recovery_actions",
|
||||
{
|
||||
id: uuid("id").primaryKey().defaultRandom(),
|
||||
companyId: uuid("company_id").notNull().references(() => companies.id),
|
||||
sourceIssueId: uuid("source_issue_id").notNull().references(() => issues.id, { onDelete: "cascade" }),
|
||||
recoveryIssueId: uuid("recovery_issue_id").references(() => issues.id, { onDelete: "set null" }),
|
||||
kind: text("kind").notNull(),
|
||||
status: text("status").notNull().default("active"),
|
||||
ownerType: text("owner_type").notNull().default("agent"),
|
||||
ownerAgentId: uuid("owner_agent_id").references(() => agents.id, { onDelete: "set null" }),
|
||||
ownerUserId: text("owner_user_id"),
|
||||
previousOwnerAgentId: uuid("previous_owner_agent_id").references(() => agents.id, { onDelete: "set null" }),
|
||||
returnOwnerAgentId: uuid("return_owner_agent_id").references(() => agents.id, { onDelete: "set null" }),
|
||||
cause: text("cause").notNull(),
|
||||
fingerprint: text("fingerprint").notNull(),
|
||||
evidence: jsonb("evidence").$type<Record<string, unknown>>().notNull().default({}),
|
||||
nextAction: text("next_action").notNull(),
|
||||
wakePolicy: jsonb("wake_policy").$type<Record<string, unknown>>(),
|
||||
monitorPolicy: jsonb("monitor_policy").$type<Record<string, unknown>>(),
|
||||
attemptCount: integer("attempt_count").notNull().default(0),
|
||||
maxAttempts: integer("max_attempts"),
|
||||
timeoutAt: timestamp("timeout_at", { withTimezone: true }),
|
||||
lastAttemptAt: timestamp("last_attempt_at", { withTimezone: true }),
|
||||
outcome: text("outcome"),
|
||||
resolutionNote: text("resolution_note"),
|
||||
resolvedAt: timestamp("resolved_at", { withTimezone: true }),
|
||||
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(table) => ({
|
||||
companySourceStatusIdx: index("issue_recovery_actions_company_source_status_idx").on(
|
||||
table.companyId,
|
||||
table.sourceIssueId,
|
||||
table.status,
|
||||
),
|
||||
companyOwnerStatusIdx: index("issue_recovery_actions_company_owner_status_idx").on(
|
||||
table.companyId,
|
||||
table.ownerAgentId,
|
||||
table.status,
|
||||
),
|
||||
companyRecoveryIssueIdx: index("issue_recovery_actions_company_recovery_issue_idx").on(
|
||||
table.companyId,
|
||||
table.recoveryIssueId,
|
||||
),
|
||||
activeSourceIdx: uniqueIndex("issue_recovery_actions_active_source_uq")
|
||||
.on(table.companyId, table.sourceIssueId)
|
||||
.where(sql`${table.status} in ('active', 'escalated')`),
|
||||
activeFingerprintIdx: uniqueIndex("issue_recovery_actions_active_fingerprint_uq")
|
||||
.on(table.companyId, table.sourceIssueId, table.cause, table.fingerprint)
|
||||
.where(sql`${table.status} in ('active', 'escalated')`),
|
||||
}),
|
||||
);
|
||||
@@ -214,6 +214,40 @@ export type IssueOriginKind = BuiltInIssueOriginKind | PluginIssueOriginKind;
|
||||
export const ISSUE_SURFACE_VISIBILITIES = ["default", "plugin_operation"] as const;
|
||||
export type IssueSurfaceVisibility = (typeof ISSUE_SURFACE_VISIBILITIES)[number];
|
||||
|
||||
export const ISSUE_RECOVERY_ACTION_KINDS = [
|
||||
"missing_disposition",
|
||||
"stranded_assigned_issue",
|
||||
"active_run_watchdog",
|
||||
"issue_graph_liveness",
|
||||
] as const;
|
||||
export type IssueRecoveryActionKind = (typeof ISSUE_RECOVERY_ACTION_KINDS)[number];
|
||||
|
||||
export const ISSUE_RECOVERY_ACTION_STATUSES = [
|
||||
"active",
|
||||
"escalated",
|
||||
"resolved",
|
||||
"cancelled",
|
||||
] as const;
|
||||
export type IssueRecoveryActionStatus = (typeof ISSUE_RECOVERY_ACTION_STATUSES)[number];
|
||||
|
||||
export const ISSUE_RECOVERY_ACTION_OWNER_TYPES = [
|
||||
"agent",
|
||||
"user",
|
||||
"board",
|
||||
"system",
|
||||
] as const;
|
||||
export type IssueRecoveryActionOwnerType = (typeof ISSUE_RECOVERY_ACTION_OWNER_TYPES)[number];
|
||||
|
||||
export const ISSUE_RECOVERY_ACTION_OUTCOMES = [
|
||||
"restored",
|
||||
"delegated",
|
||||
"false_positive",
|
||||
"blocked",
|
||||
"escalated",
|
||||
"cancelled",
|
||||
] as const;
|
||||
export type IssueRecoveryActionOutcome = (typeof ISSUE_RECOVERY_ACTION_OUTCOMES)[number];
|
||||
|
||||
export function pluginOperationIssueOriginKind(pluginKey: string): PluginIssueOriginKind {
|
||||
return `plugin:${pluginKey}:operation`;
|
||||
}
|
||||
|
||||
@@ -31,6 +31,10 @@ export {
|
||||
ISSUE_THREAD_INTERACTION_CONTINUATION_POLICIES,
|
||||
ISSUE_ORIGIN_KINDS,
|
||||
ISSUE_SURFACE_VISIBILITIES,
|
||||
ISSUE_RECOVERY_ACTION_KINDS,
|
||||
ISSUE_RECOVERY_ACTION_STATUSES,
|
||||
ISSUE_RECOVERY_ACTION_OWNER_TYPES,
|
||||
ISSUE_RECOVERY_ACTION_OUTCOMES,
|
||||
pluginOperationIssueOriginKind,
|
||||
isPluginOperationIssueOriginKind,
|
||||
ISSUE_RELATION_TYPES,
|
||||
@@ -149,6 +153,10 @@ export {
|
||||
type PluginIssueOriginKind,
|
||||
type IssueOriginKind,
|
||||
type IssueSurfaceVisibility,
|
||||
type IssueRecoveryActionKind,
|
||||
type IssueRecoveryActionStatus,
|
||||
type IssueRecoveryActionOwnerType,
|
||||
type IssueRecoveryActionOutcome,
|
||||
type IssueRelationType,
|
||||
type IssueTreeControlMode,
|
||||
type IssueTreeHoldReleasePolicyStrategy,
|
||||
@@ -373,6 +381,7 @@ export type {
|
||||
IssueBlockerAttentionState,
|
||||
IssueProductivityReview,
|
||||
IssueProductivityReviewTrigger,
|
||||
IssueRecoveryAction,
|
||||
SuccessfulRunHandoffState,
|
||||
SuccessfulRunHandoffStateKind,
|
||||
IssueScheduledRetry,
|
||||
|
||||
@@ -151,6 +151,7 @@ export type {
|
||||
IssueBlockerAttentionState,
|
||||
IssueProductivityReview,
|
||||
IssueProductivityReviewTrigger,
|
||||
IssueRecoveryAction,
|
||||
SuccessfulRunHandoffState,
|
||||
SuccessfulRunHandoffStateKind,
|
||||
IssueScheduledRetry,
|
||||
|
||||
@@ -15,6 +15,10 @@ import type {
|
||||
IssueExecutionStateStatus,
|
||||
IssueOriginKind,
|
||||
IssuePriority,
|
||||
IssueRecoveryActionKind,
|
||||
IssueRecoveryActionOutcome,
|
||||
IssueRecoveryActionOwnerType,
|
||||
IssueRecoveryActionStatus,
|
||||
IssueWorkMode,
|
||||
ModelProfileKey,
|
||||
IssueThreadInteractionContinuationPolicy,
|
||||
@@ -169,6 +173,35 @@ export interface IssueProductivityReview {
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
export interface IssueRecoveryAction {
|
||||
id: string;
|
||||
companyId: string;
|
||||
sourceIssueId: string;
|
||||
recoveryIssueId: string | null;
|
||||
kind: IssueRecoveryActionKind;
|
||||
status: IssueRecoveryActionStatus;
|
||||
ownerType: IssueRecoveryActionOwnerType;
|
||||
ownerAgentId: string | null;
|
||||
ownerUserId: string | null;
|
||||
previousOwnerAgentId: string | null;
|
||||
returnOwnerAgentId: string | null;
|
||||
cause: string;
|
||||
fingerprint: string;
|
||||
evidence: Record<string, unknown>;
|
||||
nextAction: string;
|
||||
wakePolicy: Record<string, unknown> | null;
|
||||
monitorPolicy: Record<string, unknown> | null;
|
||||
attemptCount: number;
|
||||
maxAttempts: number | null;
|
||||
timeoutAt: Date | string | null;
|
||||
lastAttemptAt: Date | string | null;
|
||||
outcome: IssueRecoveryActionOutcome | null;
|
||||
resolutionNote: string | null;
|
||||
resolvedAt: Date | string | null;
|
||||
createdAt: Date | string;
|
||||
updatedAt: Date | string;
|
||||
}
|
||||
|
||||
export type SuccessfulRunHandoffStateKind = "required" | "resolved" | "escalated";
|
||||
|
||||
export interface SuccessfulRunHandoffState {
|
||||
@@ -372,6 +405,7 @@ export interface Issue {
|
||||
blocks?: IssueRelationIssueSummary[];
|
||||
blockerAttention?: IssueBlockerAttention;
|
||||
productivityReview?: IssueProductivityReview | null;
|
||||
activeRecoveryAction?: IssueRecoveryAction | null;
|
||||
successfulRunHandoff?: SuccessfulRunHandoffState | null;
|
||||
scheduledRetry?: IssueScheduledRetry | null;
|
||||
relatedWork?: IssueRelatedWorkSummary;
|
||||
|
||||
@@ -156,6 +156,7 @@ export {
|
||||
updateIssueSchema,
|
||||
issueExecutionPolicySchema,
|
||||
issueExecutionStateSchema,
|
||||
issueRecoveryActionReadModelSchema,
|
||||
issueReviewRequestSchema,
|
||||
issueExecutionWorkspaceSettingsSchema,
|
||||
checkoutIssueSchema,
|
||||
@@ -198,6 +199,7 @@ export {
|
||||
type CreateIssueLabel,
|
||||
type UpdateIssue,
|
||||
type IssueExecutionWorkspaceSettings,
|
||||
type IssueRecoveryActionReadModel,
|
||||
type CheckoutIssue,
|
||||
type AddIssueComment,
|
||||
type CreateIssueThreadInteraction,
|
||||
|
||||
@@ -14,6 +14,10 @@ import {
|
||||
ISSUE_COMMENT_PRESENTATION_TONES,
|
||||
ISSUE_MONITOR_SCHEDULED_BY,
|
||||
ISSUE_PRIORITIES,
|
||||
ISSUE_RECOVERY_ACTION_KINDS,
|
||||
ISSUE_RECOVERY_ACTION_OUTCOMES,
|
||||
ISSUE_RECOVERY_ACTION_OWNER_TYPES,
|
||||
ISSUE_RECOVERY_ACTION_STATUSES,
|
||||
ISSUE_WORK_MODES,
|
||||
clampIssueRequestDepth,
|
||||
ISSUE_STATUSES,
|
||||
@@ -167,6 +171,37 @@ export const issueExecutionStateSchema = z.object({
|
||||
monitor: issueExecutionMonitorStateSchema.optional().nullable(),
|
||||
});
|
||||
|
||||
export const issueRecoveryActionReadModelSchema = z.object({
|
||||
id: z.string().uuid(),
|
||||
companyId: z.string().uuid(),
|
||||
sourceIssueId: z.string().uuid(),
|
||||
recoveryIssueId: z.string().uuid().nullable(),
|
||||
kind: z.enum(ISSUE_RECOVERY_ACTION_KINDS),
|
||||
status: z.enum(ISSUE_RECOVERY_ACTION_STATUSES),
|
||||
ownerType: z.enum(ISSUE_RECOVERY_ACTION_OWNER_TYPES),
|
||||
ownerAgentId: z.string().uuid().nullable(),
|
||||
ownerUserId: z.string().nullable(),
|
||||
previousOwnerAgentId: z.string().uuid().nullable(),
|
||||
returnOwnerAgentId: z.string().uuid().nullable(),
|
||||
cause: z.string().min(1),
|
||||
fingerprint: z.string().min(1),
|
||||
evidence: z.record(z.unknown()),
|
||||
nextAction: z.string().min(1),
|
||||
wakePolicy: z.record(z.unknown()).nullable(),
|
||||
monitorPolicy: z.record(z.unknown()).nullable(),
|
||||
attemptCount: z.number().int().nonnegative(),
|
||||
maxAttempts: z.number().int().positive().nullable(),
|
||||
timeoutAt: z.union([z.date(), z.string().datetime()]).nullable(),
|
||||
lastAttemptAt: z.union([z.date(), z.string().datetime()]).nullable(),
|
||||
outcome: z.enum(ISSUE_RECOVERY_ACTION_OUTCOMES).nullable(),
|
||||
resolutionNote: z.string().nullable(),
|
||||
resolvedAt: z.union([z.date(), z.string().datetime()]).nullable(),
|
||||
createdAt: z.union([z.date(), z.string().datetime()]),
|
||||
updatedAt: z.union([z.date(), z.string().datetime()]),
|
||||
});
|
||||
|
||||
export type IssueRecoveryActionReadModel = z.infer<typeof issueRecoveryActionReadModelSchema>;
|
||||
|
||||
const issueRequestDepthInputSchema = z
|
||||
.number()
|
||||
.int()
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
activityLog,
|
||||
companies,
|
||||
createDb,
|
||||
issueComments,
|
||||
issueRecoveryActions,
|
||||
issues,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
import { issueRoutes } from "../routes/issues.js";
|
||||
import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
|
||||
import { recoveryService } from "../services/recovery/service.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres issue recovery action tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
);
|
||||
}
|
||||
|
||||
describeEmbeddedPostgres("issue recovery actions", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-recovery-actions-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 30_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(issueRecoveryActions);
|
||||
await db.delete(issueComments);
|
||||
await db.delete(activityLog);
|
||||
await db.delete(issues);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function seedCompany() {
|
||||
const companyId = randomUUID();
|
||||
const managerId = randomUUID();
|
||||
const coderId = randomUUID();
|
||||
const sourceIssueId = randomUUID();
|
||||
const prefix = `RA${companyId.replaceAll("-", "").slice(0, 6).toUpperCase()}`;
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Recovery Co",
|
||||
issuePrefix: prefix,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(agents).values([
|
||||
{
|
||||
id: managerId,
|
||||
companyId,
|
||||
name: "CTO",
|
||||
role: "cto",
|
||||
status: "idle",
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
{
|
||||
id: coderId,
|
||||
companyId,
|
||||
name: "Coder",
|
||||
role: "engineer",
|
||||
status: "idle",
|
||||
reportsTo: managerId,
|
||||
adapterType: "codex_local",
|
||||
adapterConfig: {},
|
||||
runtimeConfig: {},
|
||||
permissions: {},
|
||||
},
|
||||
]);
|
||||
await db.insert(issues).values({
|
||||
id: sourceIssueId,
|
||||
companyId,
|
||||
title: "Implement backend recovery",
|
||||
status: "in_progress",
|
||||
priority: "medium",
|
||||
assigneeAgentId: coderId,
|
||||
issueNumber: 1,
|
||||
identifier: `${prefix}-1`,
|
||||
});
|
||||
const [sourceIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssueId));
|
||||
return { companyId, managerId, coderId, sourceIssueId, prefix, sourceIssue: sourceIssue! };
|
||||
}
|
||||
|
||||
it("upserts one active source-scoped action per issue and keeps company scoping explicit", async () => {
|
||||
const { companyId, managerId, sourceIssueId } = await seedCompany();
|
||||
const svc = issueRecoveryActionService(db);
|
||||
|
||||
const first = await svc.upsertSourceScoped({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "stranded_assigned_issue",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: managerId,
|
||||
cause: "stranded_assigned_issue",
|
||||
fingerprint: "recovery:fingerprint",
|
||||
evidence: { latestRunId: "run-1" },
|
||||
nextAction: "Restore a live execution path.",
|
||||
wakePolicy: { type: "wake_owner" },
|
||||
});
|
||||
const second = await svc.upsertSourceScoped({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "stranded_assigned_issue",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: managerId,
|
||||
cause: "stranded_assigned_issue",
|
||||
fingerprint: "recovery:fingerprint",
|
||||
evidence: { latestRunId: "run-2" },
|
||||
nextAction: "Restore a live execution path.",
|
||||
wakePolicy: { type: "wake_owner" },
|
||||
});
|
||||
|
||||
expect(second.id).toBe(first.id);
|
||||
expect(second.attemptCount).toBe(2);
|
||||
expect(second.evidence).toMatchObject({ latestRunId: "run-2" });
|
||||
expect(await svc.getActiveForIssue(companyId, sourceIssueId)).toMatchObject({ id: first.id });
|
||||
expect(await svc.getActiveForIssue(randomUUID(), sourceIssueId)).toBeNull();
|
||||
});
|
||||
|
||||
it("escalates stranded assigned work into a source action instead of a recovery issue", async () => {
|
||||
const { companyId, managerId, coderId, sourceIssue } = await seedCompany();
|
||||
const enqueueWakeup = vi.fn(async () => null);
|
||||
const recovery = recoveryService(db, { enqueueWakeup });
|
||||
const latestRun = {
|
||||
id: randomUUID(),
|
||||
agentId: coderId,
|
||||
status: "failed",
|
||||
error: "adapter failed",
|
||||
errorCode: "adapter_failed",
|
||||
contextSnapshot: { retryReason: "issue_continuation_needed" },
|
||||
livenessState: "needs_followup",
|
||||
} as const;
|
||||
|
||||
await recovery.escalateStrandedAssignedIssue({
|
||||
issue: sourceIssue,
|
||||
previousStatus: "in_progress",
|
||||
latestRun,
|
||||
comment: "Automatic continuation recovery failed.",
|
||||
});
|
||||
await recovery.escalateStrandedAssignedIssue({
|
||||
issue: sourceIssue,
|
||||
previousStatus: "in_progress",
|
||||
latestRun,
|
||||
comment: "Automatic continuation recovery failed.",
|
||||
});
|
||||
|
||||
const actionRows = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(eq(issueRecoveryActions.sourceIssueId, sourceIssue.id));
|
||||
expect(actionRows).toHaveLength(1);
|
||||
expect(actionRows[0]).toMatchObject({
|
||||
companyId,
|
||||
kind: "stranded_assigned_issue",
|
||||
status: "active",
|
||||
ownerAgentId: managerId,
|
||||
previousOwnerAgentId: coderId,
|
||||
returnOwnerAgentId: coderId,
|
||||
cause: "stranded_assigned_issue",
|
||||
attemptCount: 2,
|
||||
});
|
||||
|
||||
const [updatedIssue] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id));
|
||||
expect(updatedIssue).toMatchObject({
|
||||
status: "blocked",
|
||||
assigneeAgentId: managerId,
|
||||
});
|
||||
const recoveryIssues = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
|
||||
expect(recoveryIssues).toHaveLength(0);
|
||||
expect(enqueueWakeup).toHaveBeenCalledTimes(2);
|
||||
expect(enqueueWakeup.mock.calls[0]?.[1]?.payload).toMatchObject({
|
||||
issueId: sourceIssue.id,
|
||||
sourceIssueId: sourceIssue.id,
|
||||
recoveryCause: "stranded_assigned_issue",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not create nested recovery artifacts when issue-backed fallback work itself fails", async () => {
|
||||
const { companyId, managerId, sourceIssueId, prefix } = await seedCompany();
|
||||
const recoveryIssueId = randomUUID();
|
||||
await db.insert(issues).values({
|
||||
id: recoveryIssueId,
|
||||
companyId,
|
||||
title: "Recover stalled issue",
|
||||
status: "in_progress",
|
||||
priority: "medium",
|
||||
assigneeAgentId: managerId,
|
||||
parentId: sourceIssueId,
|
||||
issueNumber: 2,
|
||||
identifier: `${prefix}-2`,
|
||||
originKind: "stranded_issue_recovery",
|
||||
originId: sourceIssueId,
|
||||
originFingerprint: `stranded_issue_recovery:${sourceIssueId}`,
|
||||
});
|
||||
const [recoveryIssue] = await db.select().from(issues).where(eq(issues.id, recoveryIssueId));
|
||||
const recovery = recoveryService(db, { enqueueWakeup: vi.fn(async () => null) });
|
||||
|
||||
await recovery.escalateStrandedAssignedIssue({
|
||||
issue: recoveryIssue!,
|
||||
previousStatus: "in_progress",
|
||||
latestRun: {
|
||||
id: randomUUID(),
|
||||
agentId: managerId,
|
||||
status: "failed",
|
||||
error: "adapter failed",
|
||||
errorCode: "adapter_failed",
|
||||
contextSnapshot: { retryReason: "issue_continuation_needed" },
|
||||
livenessState: "needs_followup",
|
||||
},
|
||||
});
|
||||
|
||||
const actionRows = await db.select().from(issueRecoveryActions);
|
||||
expect(actionRows).toHaveLength(0);
|
||||
const recoveryIssues = await db
|
||||
.select()
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
|
||||
expect(recoveryIssues).toHaveLength(1);
|
||||
expect(recoveryIssues[0]?.status).toBe("blocked");
|
||||
});
|
||||
|
||||
it("exposes active recovery actions on the issue read API", async () => {
|
||||
const { companyId, managerId, sourceIssueId } = await seedCompany();
|
||||
const recoveryActionSvc = issueRecoveryActionService(db);
|
||||
const action = await recoveryActionSvc.upsertSourceScoped({
|
||||
companyId,
|
||||
sourceIssueId,
|
||||
kind: "missing_disposition",
|
||||
ownerType: "agent",
|
||||
ownerAgentId: managerId,
|
||||
cause: "successful_run_missing_issue_disposition",
|
||||
fingerprint: "missing-disposition:fingerprint",
|
||||
evidence: { sourceRunId: "run-1" },
|
||||
nextAction: "Choose a valid issue disposition.",
|
||||
wakePolicy: { type: "wake_owner" },
|
||||
});
|
||||
const app = express();
|
||||
app.use((req, _res, next) => {
|
||||
(req as any).actor = { type: "board", source: "local_implicit" };
|
||||
next();
|
||||
});
|
||||
app.use("/api", issueRoutes(db, {} as any));
|
||||
|
||||
const detail = await request(app).get(`/api/issues/${sourceIssueId}`).expect(200);
|
||||
expect(detail.body.activeRecoveryAction).toMatchObject({
|
||||
id: action.id,
|
||||
sourceIssueId,
|
||||
kind: "missing_disposition",
|
||||
ownerAgentId: managerId,
|
||||
});
|
||||
|
||||
const list = await request(app).get(`/api/issues/${sourceIssueId}/recovery-actions`).expect(200);
|
||||
expect(list.body.active).toMatchObject({ id: action.id });
|
||||
expect(list.body.actions).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -53,6 +53,7 @@ import {
|
||||
goalService,
|
||||
heartbeatService,
|
||||
issueApprovalService,
|
||||
issueRecoveryActionService,
|
||||
issueThreadInteractionService,
|
||||
ISSUE_LIST_DEFAULT_LIMIT,
|
||||
ISSUE_LIST_MAX_LIMIT,
|
||||
@@ -787,6 +788,7 @@ export function issueRoutes(
|
||||
const projectsSvc = projectService(db);
|
||||
const goalsSvc = goalService(db);
|
||||
const issueApprovalsSvc = issueApprovalService(db);
|
||||
const recoveryActionsSvc = issueRecoveryActionService(db);
|
||||
const executionWorkspacesSvc = executionWorkspaceServiceDirect(db);
|
||||
const workProductsSvc = workProductService(db);
|
||||
const documentsSvc = documentService(db);
|
||||
@@ -1541,6 +1543,7 @@ export function issueRoutes(
|
||||
attachments,
|
||||
continuationSummary,
|
||||
currentExecutionWorkspace,
|
||||
activeRecoveryAction,
|
||||
] =
|
||||
await Promise.all([
|
||||
resolveIssueProjectAndGoal(issue),
|
||||
@@ -1554,6 +1557,7 @@ export function issueRoutes(
|
||||
svc.listAttachments(issue.id),
|
||||
documentsSvc.getIssueDocumentByKey(issue.id, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY),
|
||||
currentExecutionWorkspacePromise,
|
||||
recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id),
|
||||
]);
|
||||
|
||||
res.json({
|
||||
@@ -1567,6 +1571,7 @@ export function issueRoutes(
|
||||
...(blockerAttention ? { blockerAttention } : {}),
|
||||
productivityReview,
|
||||
scheduledRetry,
|
||||
activeRecoveryAction,
|
||||
priority: issue.priority,
|
||||
projectId: issue.projectId,
|
||||
goalId: goal?.id ?? issue.goalId,
|
||||
@@ -1649,6 +1654,7 @@ export function issueRoutes(
|
||||
referenceSummary,
|
||||
successfulRunHandoffStates,
|
||||
scheduledRetry,
|
||||
activeRecoveryAction,
|
||||
] = await Promise.all([
|
||||
resolveIssueProjectAndGoal(issue),
|
||||
svc.getAncestors(issue.id),
|
||||
@@ -1660,6 +1666,7 @@ export function issueRoutes(
|
||||
issueReferencesSvc.listIssueReferenceSummary(issue.id),
|
||||
listSuccessfulRunHandoffStates(db, issue.companyId, [issue.id]),
|
||||
svc.getCurrentScheduledRetry(issue.id),
|
||||
recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id),
|
||||
]);
|
||||
const mentionedProjects = mentionedProjectIds.length > 0
|
||||
? await projectsSvc.listByIds(issue.companyId, mentionedProjectIds)
|
||||
@@ -1676,6 +1683,7 @@ export function issueRoutes(
|
||||
productivityReview,
|
||||
successfulRunHandoff: successfulRunHandoffStates.get(issue.id) ?? null,
|
||||
scheduledRetry,
|
||||
activeRecoveryAction,
|
||||
blockedBy: relations.blockedBy,
|
||||
blocks: relations.blocks,
|
||||
relatedWork: referenceSummary,
|
||||
@@ -1689,6 +1697,21 @@ export function issueRoutes(
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/issues/:id/recovery-actions", async (req, res) => {
|
||||
const id = req.params.id as string;
|
||||
const issue = await svc.getById(id);
|
||||
if (!issue) {
|
||||
res.status(404).json({ error: "Issue not found" });
|
||||
return;
|
||||
}
|
||||
assertCompanyAccess(req, issue.companyId);
|
||||
const active = await recoveryActionsSvc.getActiveForIssue(issue.companyId, issue.id);
|
||||
res.json({
|
||||
active,
|
||||
actions: active ? [active] : [],
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/issues/:id/work-products", async (req, res) => {
|
||||
const id = req.params.id as string;
|
||||
const issue = await svc.getById(id);
|
||||
|
||||
@@ -24,6 +24,7 @@ export { issueThreadInteractionService } from "./issue-thread-interactions.js";
|
||||
export { issueTreeControlService } from "./issue-tree-control.js";
|
||||
export { issueApprovalService } from "./issue-approvals.js";
|
||||
export { issueReferenceService } from "./issue-references.js";
|
||||
export { issueRecoveryActionService } from "./issue-recovery-actions.js";
|
||||
export { goalService } from "./goals.js";
|
||||
export { activityService, type ActivityFilters } from "./activity.js";
|
||||
export { approvalService } from "./approvals.js";
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { issueRecoveryActions } from "@paperclipai/db";
|
||||
import type {
|
||||
IssueRecoveryAction,
|
||||
IssueRecoveryActionKind,
|
||||
IssueRecoveryActionOwnerType,
|
||||
IssueRecoveryActionStatus,
|
||||
} from "@paperclipai/shared";
|
||||
|
||||
const ACTIVE_RECOVERY_ACTION_STATUSES = ["active", "escalated"] as const satisfies readonly IssueRecoveryActionStatus[];
|
||||
|
||||
type IssueRecoveryActionRow = typeof issueRecoveryActions.$inferSelect;
|
||||
|
||||
export type UpsertIssueRecoveryActionInput = {
|
||||
companyId: string;
|
||||
sourceIssueId: string;
|
||||
recoveryIssueId?: string | null;
|
||||
kind: IssueRecoveryActionKind;
|
||||
ownerType?: IssueRecoveryActionOwnerType;
|
||||
ownerAgentId?: string | null;
|
||||
ownerUserId?: string | null;
|
||||
previousOwnerAgentId?: string | null;
|
||||
returnOwnerAgentId?: string | null;
|
||||
cause: string;
|
||||
fingerprint: string;
|
||||
evidence?: Record<string, unknown>;
|
||||
nextAction: string;
|
||||
wakePolicy?: Record<string, unknown> | null;
|
||||
monitorPolicy?: Record<string, unknown> | null;
|
||||
maxAttempts?: number | null;
|
||||
timeoutAt?: Date | null;
|
||||
lastAttemptAt?: Date | null;
|
||||
};
|
||||
|
||||
function toReadModel(row: IssueRecoveryActionRow): IssueRecoveryAction {
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
sourceIssueId: row.sourceIssueId,
|
||||
recoveryIssueId: row.recoveryIssueId,
|
||||
kind: row.kind as IssueRecoveryAction["kind"],
|
||||
status: row.status as IssueRecoveryAction["status"],
|
||||
ownerType: row.ownerType as IssueRecoveryAction["ownerType"],
|
||||
ownerAgentId: row.ownerAgentId,
|
||||
ownerUserId: row.ownerUserId,
|
||||
previousOwnerAgentId: row.previousOwnerAgentId,
|
||||
returnOwnerAgentId: row.returnOwnerAgentId,
|
||||
cause: row.cause,
|
||||
fingerprint: row.fingerprint,
|
||||
evidence: row.evidence,
|
||||
nextAction: row.nextAction,
|
||||
wakePolicy: row.wakePolicy,
|
||||
monitorPolicy: row.monitorPolicy,
|
||||
attemptCount: row.attemptCount,
|
||||
maxAttempts: row.maxAttempts,
|
||||
timeoutAt: row.timeoutAt,
|
||||
lastAttemptAt: row.lastAttemptAt,
|
||||
outcome: row.outcome as IssueRecoveryAction["outcome"],
|
||||
resolutionNote: row.resolutionNote,
|
||||
resolvedAt: row.resolvedAt,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function isUniqueRecoveryActionConflict(error: unknown) {
|
||||
const maybe = error as { code?: string; constraint?: string; message?: string } | null;
|
||||
return Boolean(
|
||||
maybe &&
|
||||
maybe.code === "23505" &&
|
||||
(
|
||||
maybe.constraint === "issue_recovery_actions_active_source_uq" ||
|
||||
maybe.constraint === "issue_recovery_actions_active_fingerprint_uq" ||
|
||||
typeof maybe.message === "string" && (
|
||||
maybe.message.includes("issue_recovery_actions_active_source_uq") ||
|
||||
maybe.message.includes("issue_recovery_actions_active_fingerprint_uq")
|
||||
)
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function issueRecoveryActionService(db: Db) {
|
||||
async function getActiveForIssue(companyId: string, sourceIssueId: string): Promise<IssueRecoveryAction | null> {
|
||||
const row = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRecoveryActions.companyId, companyId),
|
||||
eq(issueRecoveryActions.sourceIssueId, sourceIssueId),
|
||||
inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(issueRecoveryActions.updatedAt))
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return row ? toReadModel(row) : null;
|
||||
}
|
||||
|
||||
async function listActiveForIssues(companyId: string, sourceIssueIds: string[]) {
|
||||
if (sourceIssueIds.length === 0) return new Map<string, IssueRecoveryAction>();
|
||||
const rows = await db
|
||||
.select()
|
||||
.from(issueRecoveryActions)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRecoveryActions.companyId, companyId),
|
||||
inArray(issueRecoveryActions.sourceIssueId, [...new Set(sourceIssueIds)]),
|
||||
inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(issueRecoveryActions.updatedAt));
|
||||
const result = new Map<string, IssueRecoveryAction>();
|
||||
for (const row of rows) {
|
||||
if (!result.has(row.sourceIssueId)) result.set(row.sourceIssueId, toReadModel(row));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function upsertSourceScoped(input: UpsertIssueRecoveryActionInput): Promise<IssueRecoveryAction> {
|
||||
const existing = await getActiveForIssue(input.companyId, input.sourceIssueId);
|
||||
const now = new Date();
|
||||
const ownerType = input.ownerType ?? (input.ownerAgentId ? "agent" : "board");
|
||||
if (existing) {
|
||||
const [updated] = await db
|
||||
.update(issueRecoveryActions)
|
||||
.set({
|
||||
recoveryIssueId: input.recoveryIssueId ?? null,
|
||||
kind: input.kind,
|
||||
status: "active",
|
||||
ownerType,
|
||||
ownerAgentId: input.ownerAgentId ?? null,
|
||||
ownerUserId: input.ownerUserId ?? null,
|
||||
previousOwnerAgentId: input.previousOwnerAgentId ?? existing.previousOwnerAgentId,
|
||||
returnOwnerAgentId: input.returnOwnerAgentId ?? existing.returnOwnerAgentId,
|
||||
cause: input.cause,
|
||||
fingerprint: input.fingerprint,
|
||||
evidence: input.evidence ?? existing.evidence,
|
||||
nextAction: input.nextAction,
|
||||
wakePolicy: input.wakePolicy ?? null,
|
||||
monitorPolicy: input.monitorPolicy ?? null,
|
||||
attemptCount: existing.attemptCount + 1,
|
||||
maxAttempts: input.maxAttempts ?? null,
|
||||
timeoutAt: input.timeoutAt ?? null,
|
||||
lastAttemptAt: input.lastAttemptAt ?? now,
|
||||
outcome: null,
|
||||
resolutionNote: null,
|
||||
resolvedAt: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(eq(issueRecoveryActions.id, existing.id))
|
||||
.returning();
|
||||
return toReadModel(updated!);
|
||||
}
|
||||
|
||||
try {
|
||||
const [created] = await db
|
||||
.insert(issueRecoveryActions)
|
||||
.values({
|
||||
companyId: input.companyId,
|
||||
sourceIssueId: input.sourceIssueId,
|
||||
recoveryIssueId: input.recoveryIssueId ?? null,
|
||||
kind: input.kind,
|
||||
status: "active",
|
||||
ownerType,
|
||||
ownerAgentId: input.ownerAgentId ?? null,
|
||||
ownerUserId: input.ownerUserId ?? null,
|
||||
previousOwnerAgentId: input.previousOwnerAgentId ?? null,
|
||||
returnOwnerAgentId: input.returnOwnerAgentId ?? null,
|
||||
cause: input.cause,
|
||||
fingerprint: input.fingerprint,
|
||||
evidence: input.evidence ?? {},
|
||||
nextAction: input.nextAction,
|
||||
wakePolicy: input.wakePolicy ?? null,
|
||||
monitorPolicy: input.monitorPolicy ?? null,
|
||||
attemptCount: 1,
|
||||
maxAttempts: input.maxAttempts ?? null,
|
||||
timeoutAt: input.timeoutAt ?? null,
|
||||
lastAttemptAt: input.lastAttemptAt ?? now,
|
||||
})
|
||||
.returning();
|
||||
return toReadModel(created!);
|
||||
} catch (error) {
|
||||
if (!isUniqueRecoveryActionConflict(error)) throw error;
|
||||
const raced = await getActiveForIssue(input.companyId, input.sourceIssueId);
|
||||
if (!raced) throw error;
|
||||
return upsertSourceScoped(input);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
getActiveForIssue,
|
||||
listActiveForIssues,
|
||||
upsertSourceScoped,
|
||||
};
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
issueAttachments,
|
||||
issueInboxArchives,
|
||||
issueLabels,
|
||||
issueRecoveryActions,
|
||||
issueRelations,
|
||||
issueComments,
|
||||
issueDocuments,
|
||||
@@ -1270,6 +1271,18 @@ async function listIssueBlockerAttentionMap(
|
||||
explicitWaitingIssueIds.add(parsed.issueId);
|
||||
explicitWaitingIssueIds.add(parsed.leafIssueId);
|
||||
}
|
||||
|
||||
const recoveryActionRows: Array<{ sourceIssueId: string }> = await dbOrTx
|
||||
.select({ sourceIssueId: issueRecoveryActions.sourceIssueId })
|
||||
.from(issueRecoveryActions)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRecoveryActions.companyId, companyId),
|
||||
inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
inArray(issueRecoveryActions.sourceIssueId, explicitWaitCandidateIds),
|
||||
),
|
||||
);
|
||||
for (const row of recoveryActionRows) explicitWaitingIssueIds.add(row.sourceIssueId);
|
||||
}
|
||||
|
||||
const agentRows: IssueBlockerAttentionAgentRow[] = agentIds.size > 0
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
heartbeatRunWatchdogDecisions,
|
||||
heartbeatRuns,
|
||||
issueApprovals,
|
||||
issueRecoveryActions,
|
||||
issueRelations,
|
||||
issueThreadInteractions,
|
||||
issues,
|
||||
@@ -29,6 +30,7 @@ import { redactSensitiveText } from "../../redaction.js";
|
||||
import { logActivity } from "../activity-log.js";
|
||||
import { budgetService } from "../budgets.js";
|
||||
import { instanceSettingsService } from "../instance-settings.js";
|
||||
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
|
||||
import { issueTreeControlService } from "../issue-tree-control.js";
|
||||
import { issueService } from "../issues.js";
|
||||
import { getRunLogStore } from "../run-log-store.js";
|
||||
@@ -356,6 +358,7 @@ function buildLivenessOriginalIssueComment(finding: IssueLivenessFinding, escala
|
||||
|
||||
export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) {
|
||||
const issuesSvc = issueService(db);
|
||||
const recoveryActionsSvc = issueRecoveryActionService(db);
|
||||
const treeControlSvc = issueTreeControlService(db);
|
||||
const budgets = budgetService(db);
|
||||
const instanceSettings = instanceSettingsService(db);
|
||||
@@ -1534,6 +1537,135 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
return recovery;
|
||||
}
|
||||
|
||||
function strandedRecoveryActionKind(cause: StrandedRecoveryCause) {
|
||||
return cause === SUCCESSFUL_RUN_MISSING_STATE_REASON
|
||||
? "missing_disposition" as const
|
||||
: "stranded_assigned_issue" as const;
|
||||
}
|
||||
|
||||
function strandedRecoveryActionFingerprint(input: {
|
||||
issue: typeof issues.$inferSelect;
|
||||
recoveryCause: StrandedRecoveryCause;
|
||||
latestRun: LatestIssueRun;
|
||||
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
|
||||
}) {
|
||||
const sourceRunId = input.successfulRunHandoffEvidence?.sourceRunId ?? input.latestRun?.id ?? "no-run";
|
||||
return [
|
||||
"source_scoped_recovery",
|
||||
input.issue.companyId,
|
||||
input.issue.id,
|
||||
input.recoveryCause,
|
||||
sourceRunId,
|
||||
].join(":");
|
||||
}
|
||||
|
||||
function buildStrandedRecoveryActionEvidence(input: {
|
||||
issue: typeof issues.$inferSelect;
|
||||
latestRun: LatestIssueRun;
|
||||
previousStatus: "todo" | "in_progress";
|
||||
recoveryCause: StrandedRecoveryCause;
|
||||
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
|
||||
}) {
|
||||
const context = parseObject(input.latestRun?.contextSnapshot);
|
||||
return {
|
||||
sourceIssueId: input.issue.id,
|
||||
sourceIdentifier: input.issue.identifier,
|
||||
previousStatus: input.previousStatus,
|
||||
latestIssueStatus: input.issue.status,
|
||||
latestRunId: input.latestRun?.id ?? null,
|
||||
latestRunStatus: input.latestRun?.status ?? null,
|
||||
latestRunErrorCode: input.latestRun?.errorCode ?? null,
|
||||
retryReason: readNonEmptyString(context.retryReason) ?? null,
|
||||
recoveryCause: input.recoveryCause,
|
||||
sourceRunId: input.successfulRunHandoffEvidence?.sourceRunId ?? null,
|
||||
correctiveRunId: input.successfulRunHandoffEvidence?.correctiveRunId ?? null,
|
||||
missingDisposition: input.successfulRunHandoffEvidence?.missingDisposition ?? null,
|
||||
handoffAttempt: input.successfulRunHandoffEvidence?.handoffAttempt ?? null,
|
||||
maxHandoffAttempts: input.successfulRunHandoffEvidence?.maxHandoffAttempts ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureSourceScopedStrandedRecoveryAction(input: {
|
||||
issue: typeof issues.$inferSelect;
|
||||
latestRun: LatestIssueRun;
|
||||
previousStatus: "todo" | "in_progress";
|
||||
recoveryCause?: StrandedRecoveryCause;
|
||||
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
|
||||
}) {
|
||||
const recoveryCause = input.recoveryCause ?? "stranded_assigned_issue";
|
||||
const ownerAgentId = await resolveStrandedIssueRecoveryOwnerAgentId(input.issue);
|
||||
const now = new Date();
|
||||
const action = await recoveryActionsSvc.upsertSourceScoped({
|
||||
companyId: input.issue.companyId,
|
||||
sourceIssueId: input.issue.id,
|
||||
kind: strandedRecoveryActionKind(recoveryCause),
|
||||
ownerType: ownerAgentId ? "agent" : "board",
|
||||
ownerAgentId,
|
||||
previousOwnerAgentId: input.issue.assigneeAgentId,
|
||||
returnOwnerAgentId: input.issue.assigneeAgentId,
|
||||
cause: recoveryCause,
|
||||
fingerprint: strandedRecoveryActionFingerprint({
|
||||
issue: input.issue,
|
||||
recoveryCause,
|
||||
latestRun: input.latestRun,
|
||||
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
|
||||
}),
|
||||
evidence: buildStrandedRecoveryActionEvidence({
|
||||
issue: input.issue,
|
||||
latestRun: input.latestRun,
|
||||
previousStatus: input.previousStatus,
|
||||
recoveryCause,
|
||||
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
|
||||
}),
|
||||
nextAction: recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON
|
||||
? "Choose and record a valid issue disposition without copying transcript content."
|
||||
: "Restore a live execution path, fix the runtime/adapter failure, or record an intentional manual resolution.",
|
||||
wakePolicy: ownerAgentId
|
||||
? {
|
||||
type: "wake_owner",
|
||||
reason: "source_scoped_recovery_action",
|
||||
ownerAgentId,
|
||||
}
|
||||
: {
|
||||
type: "board_escalation",
|
||||
reason: "no_invokable_recovery_owner",
|
||||
},
|
||||
monitorPolicy: null,
|
||||
maxAttempts: 1,
|
||||
lastAttemptAt: now,
|
||||
});
|
||||
|
||||
if (ownerAgentId) {
|
||||
await deps.enqueueWakeup(ownerAgentId, {
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: "source_scoped_recovery_action",
|
||||
idempotencyKey: `source_scoped_recovery_action:${action.id}:${action.attemptCount}`,
|
||||
payload: withRecoveryModelProfileHint({
|
||||
issueId: input.issue.id,
|
||||
sourceIssueId: input.issue.id,
|
||||
recoveryActionId: action.id,
|
||||
strandedRunId: input.latestRun?.id ?? null,
|
||||
recoveryCause,
|
||||
}),
|
||||
requestedByActorType: "system",
|
||||
requestedByActorId: null,
|
||||
contextSnapshot: withRecoveryModelProfileHint({
|
||||
issueId: input.issue.id,
|
||||
taskId: input.issue.id,
|
||||
wakeReason: "source_scoped_recovery_action",
|
||||
source: "issue_recovery_action",
|
||||
recoveryActionId: action.id,
|
||||
sourceIssueId: input.issue.id,
|
||||
strandedRunId: input.latestRun?.id ?? null,
|
||||
recoveryCause,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
return action;
|
||||
}
|
||||
|
||||
function buildRecoveryIssueInPlaceEscalationComment(input: {
|
||||
issue: typeof issues.$inferSelect;
|
||||
previousStatus: "todo" | "in_progress";
|
||||
@@ -1650,29 +1782,31 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
recoveryCause?: StrandedRecoveryCause;
|
||||
successfulRunHandoffEvidence?: SuccessfulRunHandoffRecoveryEvidence | null;
|
||||
}) {
|
||||
const nestedRecoverySuppressed = isStrandedIssueRecoveryIssue(input.issue);
|
||||
let recoveryIssue: typeof issues.$inferSelect | null = null;
|
||||
if (!nestedRecoverySuppressed) {
|
||||
recoveryIssue = await ensureStrandedIssueRecoveryIssue({
|
||||
if (isStrandedIssueRecoveryIssue(input.issue)) {
|
||||
return escalateStrandedRecoveryIssueInPlace({
|
||||
issue: input.issue,
|
||||
previousStatus: input.previousStatus,
|
||||
latestRun: input.latestRun,
|
||||
recoveryCause: input.recoveryCause,
|
||||
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
|
||||
});
|
||||
}
|
||||
|
||||
const recoveryAction = await ensureSourceScopedStrandedRecoveryAction({
|
||||
issue: input.issue,
|
||||
previousStatus: input.previousStatus,
|
||||
latestRun: input.latestRun,
|
||||
recoveryCause: input.recoveryCause,
|
||||
successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
|
||||
});
|
||||
const blockerIds = await existingUnresolvedBlockerIssueIds(input.issue.companyId, input.issue.id);
|
||||
const nextBlockerIds = recoveryIssue
|
||||
? [...new Set([...blockerIds, recoveryIssue.id])]
|
||||
: blockerIds;
|
||||
const updated = await issuesSvc.update(input.issue.id, {
|
||||
status: "blocked",
|
||||
blockedByIssueIds: nextBlockerIds,
|
||||
blockedByIssueIds: blockerIds,
|
||||
assigneeAgentId: recoveryAction.ownerAgentId ?? input.issue.assigneeAgentId,
|
||||
});
|
||||
if (!updated) return null;
|
||||
|
||||
const prefix = await getCompanyIssuePrefix(input.issue.companyId);
|
||||
const recoveryOwner = recoveryIssue?.assigneeAgentId ? await getAgent(recoveryIssue.assigneeAgentId) : null;
|
||||
const recoveryOwner = recoveryAction.ownerAgentId ? await getAgent(recoveryAction.ownerAgentId) : null;
|
||||
const sourceAssignee = input.issue.assigneeAgentId ? await getAgent(input.issue.assigneeAgentId) : null;
|
||||
let notice: SuccessfulRunHandoffNotice | null = null;
|
||||
if (input.recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON && input.successfulRunHandoffEvidence) {
|
||||
@@ -1683,30 +1817,26 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
: null,
|
||||
correctiveRun: input.latestRun ? { id: input.latestRun.id, status: input.latestRun.status } : null,
|
||||
sourceAssignee,
|
||||
recoveryIssue,
|
||||
recoveryIssue: null,
|
||||
recoveryOwner,
|
||||
latestIssueStatus: input.issue.status,
|
||||
latestHandoffRunStatus: input.latestRun?.status ?? "unknown",
|
||||
missingDisposition: input.successfulRunHandoffEvidence.missingDisposition,
|
||||
});
|
||||
}
|
||||
let recoveryLine: string;
|
||||
if (nestedRecoverySuppressed) {
|
||||
recoveryLine = await buildNestedStrandedRecoveryLine(input.issue, prefix);
|
||||
} else if (recoveryIssue) {
|
||||
recoveryLine = [
|
||||
const recoveryLine = recoveryAction.ownerAgentId
|
||||
? [
|
||||
"",
|
||||
`- Recovery issue: ${issueUiLink({ identifier: recoveryIssue.identifier, id: recoveryIssue.id }, prefix)}`,
|
||||
`- Recovery action: \`${recoveryAction.id}\``,
|
||||
`- Recovery owner: ${agentUiLink(recoveryOwner, prefix)}`,
|
||||
"- Next action: the recovery owner should either restore a live execution path or record the manual resolution, then mark the recovery issue done.",
|
||||
].join("\n");
|
||||
} else {
|
||||
recoveryLine = [
|
||||
"- Next action: the recovery owner should either restore a live execution path or record the manual resolution on the source issue.",
|
||||
].join("\n")
|
||||
: [
|
||||
"",
|
||||
"- Recovery issue: none created because Paperclip could not find an invokable manager, creator, or executive owner with budget available.",
|
||||
`- Recovery action: \`${recoveryAction.id}\``,
|
||||
"- Recovery owner: board escalation, because Paperclip could not find an invokable manager, creator, or executive owner with budget available.",
|
||||
"- Next action: a board operator should assign an invokable recovery owner, fix the agent/runtime state, or record an intentional manual resolution.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (notice) {
|
||||
await issuesSvc.addComment(input.issue.id, notice.body, {}, {
|
||||
@@ -1740,9 +1870,11 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
latestRunId: input.latestRun?.id ?? null,
|
||||
latestRunStatus: input.latestRun?.status ?? null,
|
||||
latestRunErrorCode: input.latestRun?.errorCode ?? null,
|
||||
recoveryIssueId: recoveryIssue?.id ?? null,
|
||||
nestedRecoverySuppressed,
|
||||
blockerIssueIds: nextBlockerIds,
|
||||
recoveryActionId: recoveryAction.id,
|
||||
recoveryOwnerAgentId: recoveryAction.ownerAgentId,
|
||||
previousOwnerAgentId: recoveryAction.previousOwnerAgentId,
|
||||
returnOwnerAgentId: recoveryAction.returnOwnerAgentId,
|
||||
blockerIssueIds: blockerIds,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -2016,6 +2148,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
interactionRows,
|
||||
approvalRows,
|
||||
recoveryIssueRows,
|
||||
recoveryActionRows,
|
||||
] = await Promise.all([
|
||||
db
|
||||
.select({
|
||||
@@ -2132,6 +2265,14 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
notInArray(issues.status, ["done", "cancelled"]),
|
||||
),
|
||||
),
|
||||
db
|
||||
.select({
|
||||
companyId: issueRecoveryActions.companyId,
|
||||
issueId: issueRecoveryActions.sourceIssueId,
|
||||
status: issueRecoveryActions.status,
|
||||
})
|
||||
.from(issueRecoveryActions)
|
||||
.where(inArray(issueRecoveryActions.status, ["active", "escalated"])),
|
||||
]);
|
||||
|
||||
const openRecoveryIssues = recoveryIssueRows.flatMap((row) => {
|
||||
@@ -2185,7 +2326,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
|
||||
})),
|
||||
pendingInteractions: interactionRows,
|
||||
pendingApprovals: approvalRows,
|
||||
openRecoveryIssues,
|
||||
openRecoveryIssues: openRecoveryIssues.concat(recoveryActionRows),
|
||||
now: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user