PaperClipAI/server/src/__tests__/recovery-observability.test.ts

import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
  agents,
  companies,
  createDb,
  heartbeatRuns,
  issueRecoveryActions,
  issues,
} from "@paperclipai/db";
import {
  getEmbeddedPostgresTestSupport,
  startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import {
  classifyRecoveryHandoff,
  evaluateRecoveryRateAlert,
  MAX_WINDOW_WEEKS,
  recoveryObservabilityService,
  type WeeklyRecoveryRate,
} from "../services/recovery-observability.ts";

const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;

if (!embeddedPostgresSupport.supported) {
  console.warn(
    `Skipping embedded Postgres recovery observability tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
  );
}

function week(runs: number, recoveryActions: number, weekStart: string): WeeklyRecoveryRate {
  return {
    weekStart,
    runs,
    recoveryActions,
    ratePercent: runs > 0 ? Math.round((recoveryActions / runs) * 10000) / 100 : 0,
  };
}

describe("evaluateRecoveryRateAlert", () => {
  it("fires when a week crosses the 2% threshold", () => {
    const weekly = [
      week(1000, 12, "2026-06-01"), // 1.20%
      week(2364, 77, "2026-07-06"), // 3.26% — the regression week
      week(1000, 10, "2026-07-13"), // 1.00%
    ];

    const alert = evaluateRecoveryRateAlert(weekly, 2);

    expect(alert.breached).toBe(true);
    expect(alert.breachedWeeks.map((w) => w.weekStart)).toEqual(["2026-07-06"]);
    expect(alert.latestWeek?.weekStart).toBe("2026-07-13");
    expect(alert.latestWeekBreached).toBe(false);
  });

  it("stays quiet when every week is under the threshold", () => {
    const weekly = [week(1000, 13, "2026-06-01"), week(1000, 19, "2026-06-08")];
    const alert = evaluateRecoveryRateAlert(weekly, 2);
    expect(alert.breached).toBe(false);
    expect(alert.breachedWeeks).toHaveLength(0);
    expect(alert.latestWeekBreached).toBe(false);
  });

  it("flags the latest week when it is the one that regresses", () => {
    const weekly = [week(1000, 10, "2026-07-06"), week(1000, 30, "2026-07-13")]; // 1% then 3%
    const alert = evaluateRecoveryRateAlert(weekly, 2);
    expect(alert.latestWeekBreached).toBe(true);
  });
});

describe("classifyRecoveryHandoff", () => {
  const base = {
    status: "resolved",
    outcome: "restored",
    ownerAgentId: "manager",
    returnOwnerAgentId: "coder",
    finalAssigneeAgentId: "coder",
    finalIssueStatus: "done",
  };

  it("marks a manager who kept and completed the work as owner_completed", () => {
    expect(
      classifyRecoveryHandoff({ ...base, finalAssigneeAgentId: "manager", finalIssueStatus: "done" }),
    ).toBe("owner_completed");
  });

  it("does not treat in-review work as a completed owner handoff", () => {
    expect(
      classifyRecoveryHandoff({ ...base, finalAssigneeAgentId: "manager", finalIssueStatus: "in_review" }),
    ).toBe("other");
  });

  it("marks work returned to the original assignee as handed_back", () => {
    expect(classifyRecoveryHandoff({ ...base, finalAssigneeAgentId: "coder" })).toBe("handed_back");
  });

  it("marks work delegated to a different specialist as handed_back", () => {
    expect(classifyRecoveryHandoff({ ...base, finalAssigneeAgentId: "other-agent" })).toBe(
      "handed_back",
    );
  });

  it("marks the original agent recovering its own issue as self_recovery", () => {
    expect(
      classifyRecoveryHandoff({ ...base, ownerAgentId: "coder", returnOwnerAgentId: "coder" }),
    ).toBe("self_recovery");
  });

  it("treats still-active actions as active regardless of assignee", () => {
    expect(classifyRecoveryHandoff({ ...base, status: "active" })).toBe("active");
  });
});

describeEmbeddedPostgres("recovery observability report", () => {
  let db!: ReturnType<typeof createDb>;
  let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;

  const now = new Date("2026-07-15T12:00:00.000Z");
  // Monday of each relevant week (matches Postgres date_trunc('week', ...)).
  const regressionWeek = new Date("2026-07-08T12:00:00.000Z"); // in week 2026-07-06
  const latestWeek = new Date("2026-07-14T12:00:00.000Z"); // in week 2026-07-13

  beforeAll(async () => {
    tempDb = await startEmbeddedPostgresTestDatabase("paperclip-recovery-observability-");
    db = createDb(tempDb.connectionString);
  }, 20_000);

  afterEach(async () => {
    await db.delete(issueRecoveryActions);
    await db.delete(heartbeatRuns);
    await db.delete(issues);
    await db.delete(agents);
    await db.delete(companies);
  });

  afterAll(async () => {
    await tempDb?.cleanup();
  });

  async function seedBaseline() {
    const companyId = randomUUID();
    const managerId = randomUUID();
    const coderId = randomUUID();
    const otherId = randomUUID();

    await db.insert(companies).values({
      id: companyId,
      name: "Paperclip",
      issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
      requireBoardApprovalForNewAgents: false,
    });

    await db.insert(agents).values(
      [managerId, coderId, otherId].map((id, index) => ({
        id,
        companyId,
        name: `agent-${index}`,
        role: "engineer",
        status: "running",
        adapterType: "codex_local",
        adapterConfig: {},
        runtimeConfig: {},
        permissions: {},
      })),
    );

    // 100 runs in the regression week, 100 in the latest week.
    await db.insert(heartbeatRuns).values([
      ...Array.from({ length: 100 }, () => ({
        id: randomUUID(),
        companyId,
        agentId: coderId,
        invocationSource: "assignment",
        status: "succeeded",
        createdAt: regressionWeek,
      })),
      ...Array.from({ length: 100 }, () => ({
        id: randomUUID(),
        companyId,
        agentId: coderId,
        invocationSource: "assignment",
        status: "succeeded",
        createdAt: latestWeek,
      })),
    ]);

    return { companyId, managerId, coderId, otherId };
  }

  async function seedRecoveryAction(input: {
    companyId: string;
    n: number;
    createdAt: Date;
    cause: string;
    errorCode: string;
    status: string;
    outcome: string | null;
    ownerAgentId: string | null;
    returnOwnerAgentId: string | null;
    finalAssigneeAgentId: string | null;
    finalIssueStatus: string;
  }) {
    const sourceIssueId = randomUUID();
    await db.insert(issues).values({
      id: sourceIssueId,
      companyId: input.companyId,
      title: `Source ${input.n}`,
      status: input.finalIssueStatus,
      priority: "medium",
      assigneeAgentId: input.finalAssigneeAgentId,
      issueNumber: input.n,
      identifier: `SRC-${input.n}`,
    });
    await db.insert(issueRecoveryActions).values({
      id: randomUUID(),
      companyId: input.companyId,
      sourceIssueId,
      kind: "stranded_assigned_issue",
      status: input.status,
      ownerType: input.ownerAgentId ? "agent" : "board",
      ownerAgentId: input.ownerAgentId,
      returnOwnerAgentId: input.returnOwnerAgentId,
      previousOwnerAgentId: input.returnOwnerAgentId,
      cause: input.cause,
      fingerprint: `fp-${input.n}`,
      evidence: { latestRunErrorCode: input.errorCode },
      nextAction: "recover",
      outcome: input.outcome,
      createdAt: input.createdAt,
      updatedAt: input.createdAt,
    });
  }

  it("fires the threshold alert on synthetic data crossing 2% of runs", async () => {
    const { companyId, managerId, coderId } = await seedBaseline();

    // Regression week: 3 recovery actions against 100 runs => 3% > 2%.
    for (let i = 0; i < 3; i += 1) {
      await seedRecoveryAction({
        companyId,
        n: i + 1,
        createdAt: regressionWeek,
        cause: "stranded_assigned_issue",
        errorCode: "process_lost",
        status: "resolved",
        outcome: "restored",
        ownerAgentId: managerId,
        returnOwnerAgentId: coderId,
        finalAssigneeAgentId: managerId,
        finalIssueStatus: "done",
      });
    }
    // Latest week: 1 recovery action against 100 runs => 1% (under threshold).
    await seedRecoveryAction({
      companyId,
      n: 99,
      createdAt: latestWeek,
      cause: "stranded_assigned_issue",
      errorCode: "process_lost",
      status: "resolved",
      outcome: "restored",
      ownerAgentId: coderId,
      returnOwnerAgentId: coderId,
      finalAssigneeAgentId: coderId,
      finalIssueStatus: "done",
    });

    const report = await recoveryObservabilityService(db).report(companyId, {
      now,
      weeks: 8,
      thresholdPercent: 2,
    });

    expect(report.alert.breached).toBe(true);
    expect(report.alert.breachedWeeks.map((w) => w.weekStart)).toEqual(["2026-07-06"]);
    const regression = report.weekly.find((w) => w.weekStart === "2026-07-06");
    expect(regression).toMatchObject({ runs: 100, recoveryActions: 3, ratePercent: 3 });
    const latest = report.weekly.find((w) => w.weekStart === "2026-07-13");
    expect(latest).toMatchObject({ runs: 100, recoveryActions: 1, ratePercent: 1 });
    expect(report.alert.latestWeekBreached).toBe(false);
  });

  it("computes the handed_back vs owner_completed ratio and per-cause routing", async () => {
    const { companyId, managerId, coderId, otherId } = await seedBaseline();

    // Two manager takeovers where the manager kept and completed the work.
    for (let i = 0; i < 2; i += 1) {
      await seedRecoveryAction({
        companyId,
        n: i + 1,
        createdAt: regressionWeek,
        cause: "stranded_assigned_issue",
        errorCode: "adapter_failed",
        status: "resolved",
        outcome: "restored",
        ownerAgentId: managerId,
        returnOwnerAgentId: coderId,
        finalAssigneeAgentId: managerId,
        finalIssueStatus: "done",
      });
    }
    // One takeover handed back to the original assignee.
    await seedRecoveryAction({
      companyId,
      n: 3,
      createdAt: regressionWeek,
      cause: "stranded_assigned_issue",
      errorCode: "adapter_failed",
      status: "resolved",
      outcome: "restored",
      ownerAgentId: managerId,
      returnOwnerAgentId: coderId,
      finalAssigneeAgentId: coderId,
      finalIssueStatus: "in_progress",
    });
    // One handed to a different specialist (still counts as handed back).
    await seedRecoveryAction({
      companyId,
      n: 4,
      createdAt: regressionWeek,
      cause: "stranded_assigned_issue",
      errorCode: "adapter_failed",
      status: "resolved",
      outcome: "restored",
      ownerAgentId: managerId,
      returnOwnerAgentId: coderId,
      finalAssigneeAgentId: otherId,
      finalIssueStatus: "in_progress",
    });
    // Original agent recovered its own process_lost issue (self recovery).
    await seedRecoveryAction({
      companyId,
      n: 5,
      createdAt: regressionWeek,
      cause: "process_lost",
      errorCode: "process_lost",
      status: "resolved",
      outcome: "restored",
      ownerAgentId: coderId,
      returnOwnerAgentId: coderId,
      finalAssigneeAgentId: coderId,
      finalIssueStatus: "done",
    });
    // Escalated actions are still active, but they need their own routing counter.
    await seedRecoveryAction({
      companyId,
      n: 6,
      createdAt: regressionWeek,
      cause: "stranded_assigned_issue",
      errorCode: "manual_escalation",
      status: "escalated",
      outcome: null,
      ownerAgentId: managerId,
      returnOwnerAgentId: coderId,
      finalAssigneeAgentId: managerId,
      finalIssueStatus: "in_progress",
    });

    const report = await recoveryObservabilityService(db).report(companyId, { now, weeks: 8 });

    expect(report.handoff.ownerCompleted).toBe(2);
    expect(report.handoff.handedBack).toBe(2);
    expect(report.handoff.selfRecovery).toBe(1);
    expect(report.handoff.resolvedTakeovers).toBe(4);
    expect(report.handoff.handedBackRatio).toBe(50);
    expect(report.handoff.ownerCompletedRatio).toBe(50);

    const strandedCause = report.byCause.find(
      (c) => c.cause === "stranded_assigned_issue" && c.latestRunErrorCode === "adapter_failed",
    );
    expect(strandedCause?.count).toBe(4);

    const processLostRouting = report.perCauseRouting.find((r) => r.cause === "process_lost");
    expect(processLostRouting?.retriedByOriginalSucceeded).toBe(1);
    const strandedRouting = report.perCauseRouting.find(
      (r) => r.cause === "stranded_assigned_issue",
    );
    expect(strandedRouting?.ownerCompleted).toBe(2);
    expect(strandedRouting?.handedBack).toBe(2);
    expect(strandedRouting?.active).toBe(1);
    expect(strandedRouting?.escalated).toBe(1);
  });

  it("caps the reporting window so a huge `weeks` value can't over-allocate", async () => {
    const { companyId } = await seedBaseline();

    const report = await recoveryObservabilityService(db).report(companyId, {
      now,
      weeks: 100_000,
    });

    expect(report.window.weeks).toBe(MAX_WINDOW_WEEKS);
    expect(report.window.since).toBe(report.weekly[0]?.weekStart);
    expect(report.window.since).not.toContain("T");
    expect(report.weekly).toHaveLength(MAX_WINDOW_WEEKS);
  });

  it("counts an active board recovery action without reporting an active takeover", async () => {
    const { companyId, coderId } = await seedBaseline();
    await seedRecoveryAction({
      companyId,
      n: 101,
      createdAt: latestWeek,
      cause: "process_lost",
      errorCode: "process_lost",
      status: "active",
      outcome: null,
      ownerAgentId: null,
      returnOwnerAgentId: coderId,
      finalAssigneeAgentId: coderId,
      finalIssueStatus: "blocked",
    });

    const report = await recoveryObservabilityService(db).report(companyId, { now, weeks: 8 });

    expect(report.handoff).toMatchObject({ boardOwned: 1, activeTakeovers: 0 });
    expect(report.perCauseRouting.find((entry) => entry.cause === "process_lost")).toMatchObject({
      active: 1,
    });
  });
});