PaperClipAI/server/src/__tests__/decision-effect-recovery-work.test.ts

import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { activityLog, agents, authUsers, companies, companyMemberships, createDb, decisionEffectExecutions,
  decisions, decisionRetention, decisionTargetIssues, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { decisionService } from "../services/decisions.js";
import { registerDecisionEffectRecoveryWork } from "../services/decision-effect-recovery-work.js";
import { registerDecisionContinuationWork } from "../services/decision-continuation-work.js";
import { writeDecisionChange } from "../services/decision-work-notifications.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { hashAttentionArchiveManifest } from "../services/decision-retention.js";
import type { AttentionArchiveManifestEntry } from "@paperclipai/shared";
import { idleWorkSnapshot } from "../services/task-admission.js";

const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_DECISION_EFFECTS_TEST_DATABASE_URL");
const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("decision effect recovery worker", () => {
  let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
  let db: ReturnType<typeof createDb>;
  let companyId: string, agentId: string, originId: string, targetId: string, runId: string, userId: string;
  let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
  let draining: boolean;
  let recovery: ReturnType<typeof decisionService>;
  const errors: unknown[] = [];
  const wake = vi.fn(async () => {});
  const service = () => decisionService(db, { wakeOriginAgent: wake });

  beforeAll(async () => {
    if (!externalUrl) temporary = await startEmbeddedPostgresTestDatabase("decision-effects-");
    db = createDb(externalUrl ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
  }, 30_000);
  beforeEach(async () => {
    vi.stubEnv("PAPERCLIP_DECISION_SIGNING_SECRET", "0123456789abcdef0123456789abcdef");
    companyId = randomUUID(); agentId = randomUUID(); originId = randomUUID(); targetId = randomUUID(); runId = randomUUID(); userId = randomUUID();
    await db.insert(companies).values({ id: companyId, name: "Recovery", issuePrefix: `R${companyId.slice(0, 6)}`, requireBoardApprovalForNewAgents: false });
    await db.insert(authUsers).values({ id: userId, name: "Decider", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() });
    await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "member" });
    await db.insert(agents).values({ id: agentId, companyId, name: "Proposer", role: "engineer", status: "active", adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {} });
    await db.insert(issues).values([
      { id: originId, companyId, title: "Origin", status: "in_progress", priority: "medium", assigneeAgentId: agentId, responsibleUserId: userId },
      { id: targetId, companyId, title: "Target", status: "todo", priority: "medium", responsibleUserId: userId },
    ]);
    await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", responsibleUserId: userId, contextSnapshot: { issueId: originId } });
    await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
    draining = false; errors.length = 0; wake.mockReset();
    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true });
  });
  afterEach(async () => {
    await coordinator?.stop(); coordinator = undefined;
    vi.useRealTimers(); vi.restoreAllMocks(); vi.unstubAllEnvs();
    await db.execute(sql`drop trigger if exists decision_effect_finish_fail on decisions`);
    await db.execute(sql`drop function if exists decision_effect_finish_fail()`);
    await db.delete(decisionEffectExecutions); await db.delete(decisionTargetIssues); await db.delete(decisions);
    await db.delete(decisionRetention); await db.delete(activityLog); await db.delete(issueComments); await db.delete(heartbeatRuns);
    await db.delete(issues); await db.delete(agents); await db.delete(companyMemberships); await db.delete(authUsers); await db.delete(companies);
    expect(errors).toEqual([]);
  });
  afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });

  const actor = () => ({ type: "agent" as const, companyId, agentId, runId, source: "agent_jwt" as const,
    onBehalfOfUserId: userId, onBehalfOfMemberships: [{ companyId, membershipRole: "member", status: "active" }] });
  const board = () => ({ type: "board" as const, userId, companyIds: [companyId], source: "session" as const,
    memberships: [{ companyId, membershipRole: "member", status: "active" }] });
  const create = () => service().create({ companyId, agentId, runId, actor: actor(), title: "Comment?", body: "Body",
    continuationPolicy: "wake_origin_agent", options: [{ id: "yes", label: "Yes", effects: [
      { type: "comment_on_issue", targetIssueId: targetId, staleness: "lenient", bodyMarkdown: "hello" },
    ] }] });
  const decide = (id: string) => service().decide({ id, optionId: "yes", decidedByUserId: userId, userActor: board() });
  const claim = (id: string, updatedAt = new Date(), owner = db) => writeDecisionChange(owner, async tx => tx.update(decisions).set({
    status: "decided", executionStatus: "running", chosenOptionId: "yes", decidedByUserId: userId,
    updatedAt, inputValues: {}, metadata: { continuationPending: true },
  }).where(eq(decisions.id, id)));
  async function start() {
    coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
      onError: error => errors.push(error) });
    await registerDecisionContinuationWork(coordinator, service(), () => !draining).ready;
    recovery = service();
    vi.spyOn(recovery, "recoverEffects");
    await registerDecisionEffectRecoveryWork(coordinator, recovery, () => !draining).ready;
  }
  async function quiet() {
    await vi.waitFor(async () => {
      await vi.advanceTimersByTimeAsync(30_000);
      expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull();
    }, { timeout: 5000 });
    const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "update"), executes = vi.spyOn(db, "execute");
    await vi.advanceTimersByTimeAsync(60 * 60_000);
    expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled();
    reads.mockRestore(); writes.mockRestore(); executes.mockRestore();
  }
  async function settled(id: string, executionStatus = "succeeded") {
    await vi.waitFor(async () => {
      await vi.advanceTimersByTimeAsync(30_000);
      expect(await service().get(id)).toMatchObject({ executionStatus, metadata: { continuationPending: false } });
    }, { timeout: 5000 });
  }
  const comments = () => db.select().from(issueComments).where(eq(issueComments.issueId, targetId));
  async function failSettlement() {
    await db.execute(sql`create function decision_effect_finish_fail() returns trigger language plpgsql as $$ begin
      if OLD.execution_status = 'running' and NEW.execution_status <> 'running' then raise exception 'settlement unavailable'; end if;
      return NEW; end $$`);
    await db.execute(sql`create trigger decision_effect_finish_fail before update on decisions for each row execute function decision_effect_finish_fail()`);
  }

  it("stays quiet with open decisions and delivered history after a real decision", async () => {
    await start(); await quiet(); const row = await create(); await quiet();
    await decide(row.id); await quiet(); expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(1);
  });

  it("reactivates from the actual claim and recovers a failed settlement without repeating the effect", async () => {
    await start(); await quiet(); const row = await create(); await failSettlement();
    await expect(decide(row.id)).rejects.toMatchObject({ cause: { message: "settlement unavailable" } });
    expect((await service().get(row.id))?.executionStatus).toBe("running"); expect(await comments()).toHaveLength(1);
    await db.execute(sql`drop trigger decision_effect_finish_fail on decisions`);
    await settled(row.id); await quiet(); expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(1);
  });

  it("reconstructs the saved grace deadline at restart and recovers a claimed effect", async () => {
    const row = await create(); const claimedAt = new Date(); await claim(row.id, claimedAt);
    await db.insert(decisionEffectExecutions).values({ decisionId: row.id, effectIndex: 0, effectType: "comment_on_issue", targetIssueId: targetId, status: "claimed" });
    await start(); expect(coordinator!.nextWakeAt()).toBe(claimedAt.getTime() + 60_000);
    await vi.advanceTimersByTimeAsync(30_000); expect(await comments()).toHaveLength(0);
    await settled(row.id); await quiet(); expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(1);
  });

  it("recovers at the exact grace boundary and preserves a zero grace override", async () => {
    const row = await create(), at = new Date(); await claim(row.id, at);
    expect(await service().recoverEffects(new Date(at.getTime() + 59_999))).toEqual({ resumed: 0 });
    expect(await service().recoverEffects(new Date(at.getTime() + 60_000))).toEqual({ resumed: 1 });
    vi.stubEnv("PAPERCLIP_DECISIONS_RECOVERY_GRACE_MS", "0");
    const second = await create(); await claim(second.id, at);
    expect(await service().recoverEffects(at)).toEqual({ resumed: 1 }); expect(await comments()).toHaveLength(2);
  });

  it("joins active foreground effects when their recovery deadline arrives", async () => {
    const row = await create(); await start();
    let release!: () => void, locked!: () => void;
    const gate = new Promise<void>(resolve => { release = resolve; });
    const acquired = new Promise<void>(resolve => { locked = resolve; });
    const lock = db.transaction(async tx => {
      await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`decision-effect:${row.id}:0`}, 0))`);
      locked(); await gate;
    });
    await acquired;
    const deciding = decide(row.id);
    try {
      await vi.waitFor(async () => expect((await service().get(row.id))?.executionStatus).toBe("running"));
      await vi.advanceTimersByTimeAsync(120_000);
      expect(await comments()).toHaveLength(0); expect(wake).not.toHaveBeenCalled();
      await vi.waitFor(() => expect(recovery.recoverEffects).toHaveBeenCalledTimes(2));
    } finally { release(); }
    await lock; await deciding; await quiet();
    expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(1);
  });

  it("defers an outer transaction until commit and ignores rollback", async () => {
    const row = await create(); await start(); await quiet();
    await expect(db.transaction(async tx => {
      await claim(row.id, new Date(Date.now() - 120_000), tx as unknown as typeof db);
      await vi.advanceTimersByTimeAsync(120_000); expect(await comments()).toHaveLength(0); throw new Error("rollback");
    })).rejects.toThrow("rollback");
    await quiet(); expect((await service().get(row.id))?.status).toBe("open");
    await db.transaction(async tx => {
      await claim(row.id, new Date(Date.now() - 120_000), tx as unknown as typeof db);
      await vi.advanceTimersByTimeAsync(120_000); expect(await comments()).toHaveLength(0);
    });
    await settled(row.id); await quiet(); expect(await comments()).toHaveLength(1);
  });

  it("retains a first claim through drain without passive SQL", async () => {
    const row = await create(); await start(); await quiet(); draining = true;
    await claim(row.id, new Date(Date.now() - 120_000));
    const reads = vi.spyOn(db, "select"); await vi.advanceTimersByTimeAsync(120_000);
    expect(reads).not.toHaveBeenCalled(); reads.mockRestore();
    draining = false; await settled(row.id); await quiet(); expect(await comments()).toHaveLength(1);
  });

  it("retains a failed recovery through drain and does not replay the completed effect", async () => {
    const row = await create(); await claim(row.id, new Date(Date.now() - 120_000)); await failSettlement(); await start();
    await vi.waitFor(() => expect(errors.length).toBeGreaterThan(0)); draining = true;
    expect(await comments()).toHaveLength(1);
    const reads = vi.spyOn(db, "select"); await vi.advanceTimersByTimeAsync(120_000);
    expect(reads).not.toHaveBeenCalled(); reads.mockRestore();
    await db.execute(sql`drop trigger decision_effect_finish_fail on decisions`);
    draining = false; await settled(row.id); await quiet(); expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(1);
    for (const error of errors) expect(error).toMatchObject({ cause: { message: "settlement unavailable" } }); errors.length = 0;
  });

  it("rechecks current membership before executing a recovered effect", async () => {
    const row = await create(); await claim(row.id, new Date(Date.now() - 120_000));
    await db.update(companyMemberships).set({ status: "inactive" }).where(eq(companyMemberships.principalId, userId));
    await start(); await settled(row.id, "failed"); await quiet(); expect(await comments()).toHaveLength(0);
    const result = await service().outcome(row.id);
    expect(result.executions[0]?.error).toBe("deny_decision_intersection"); expect(wake).toHaveBeenCalledTimes(1);
  });

  it("leaves continuation failures to their owner after effect recovery finishes", async () => {
    const row = await create(); await claim(row.id, new Date(Date.now() - 120_000));
    wake.mockRejectedValueOnce(new Error("wake unavailable")); await start(); await settled(row.id); await quiet();
    expect(await comments()).toHaveLength(1); expect(wake).toHaveBeenCalledTimes(2);
    expect(errors.every(error => String(error).includes("wake unavailable"))).toBe(true); errors.length = 0;
  });

  it("keeps aggregate maintenance from recovering effects when the worker owns them", async () => {
    const row = await create(); await claim(row.id, new Date(Date.now() - 120_000));
    expect(await service().sweepExpired(new Date(), { includeContinuations: false, includeExpiry: false, includeEffects: false }))
      .toEqual({ expired: 0, resumed: 0 });
    expect(await comments()).toHaveLength(0);
    expect(await service().sweepExpired()).toEqual({ expired: 0, resumed: 1 }); expect(await comments()).toHaveLength(1);
  });

  it("pages past invalid legacy claims without inventing authority or losing their debt", async () => {
    vi.stubEnv("PAPERCLIP_DECISIONS_SWEEP_BATCH_SIZE", "1");
    const invalid = await create(), valid = await create();
    await claim(invalid.id, new Date(Date.now() - 180_000)); await claim(valid.id, new Date(Date.now() - 120_000));
    await db.update(decisions).set({ decidedByUserId: null }).where(eq(decisions.id, invalid.id));
    const recovery = service();
    expect(await recovery.recoverEffects()).toEqual({ resumed: 0 }); expect(await recovery.nextEffectRecoveryAt()).not.toBeNull();
    expect(await recovery.recoverEffects()).toEqual({ resumed: 1 }); expect(await comments()).toHaveLength(1);
    expect((await recovery.get(invalid.id))?.executionStatus).toBe("running");
    expect(await recovery.nextEffectRecoveryAt()).not.toBeNull();
  });

  it("pages past invalid claims using database timestamp precision", async () => {
    vi.stubEnv("PAPERCLIP_DECISIONS_SWEEP_BATCH_SIZE", "1");
    const invalid = await create(), valid = await create();
    await claim(invalid.id); await claim(valid.id, new Date(Date.now() - 120_000));
    await db.execute(sql`update decisions set decided_by_user_id = null,
      updated_at = date_trunc('milliseconds', now() - interval '3 minutes') + interval '456 microseconds'
      where id = ${invalid.id}`);
    const recovery = service();
    expect(await recovery.recoverEffects()).toEqual({ resumed: 0 });
    expect(await recovery.recoverEffects()).toEqual({ resumed: 1 });
    expect(await comments()).toHaveLength(1);
    expect((await recovery.get(invalid.id))?.executionStatus).toBe("running");
    expect(await recovery.nextEffectRecoveryAt()).not.toBeNull();
    expect(await recovery.recoverEffects()).toEqual({ resumed: 0 });
    expect(await recovery.recoverEffects()).toEqual({ resumed: 0 });
    expect(await comments()).toHaveLength(1);
  });

  it("advances past malformed claims and retries them on a later pass", async () => {
    vi.stubEnv("PAPERCLIP_DECISIONS_SWEEP_BATCH_SIZE", "1");
    const invalid = await create(), valid = await create();
    await claim(invalid.id, new Date(Date.now() - 180_000)); await claim(valid.id, new Date(Date.now() - 120_000));
    await db.update(decisions).set({ chosenOptionId: "missing" }).where(eq(decisions.id, invalid.id));
    const recovery = service();
    await expect(recovery.recoverEffects()).rejects.toThrow("Stored decision outcome is invalid");
    expect(await recovery.recoverEffects()).toEqual({ resumed: 1 }); expect(await comments()).toHaveLength(1);
    await recovery.recoverEffects();
    await expect(recovery.recoverEffects()).rejects.toThrow("Stored decision outcome is invalid");
  });

  it.each([false, true])("preserves archive recovery semantics with prior committed archive: %s", async committed => {
    await db.update(issues).set({ status: "in_review", assigneeUserId: userId }).where(eq(issues.id, targetId));
    const activityAt = new Date("2026-04-01T00:00:00.000Z");
    await db.insert(decisionRetention).values({ companyId, sourceKind: "review", sourceId: targetId, sourceActivityAt: activityAt });
    const manifest: AttentionArchiveManifestEntry[] = [{ companyId, sourceKind: "review", sourceId: targetId,
      expectedVersion: 1, activityAt: activityAt.toISOString(), reason: "Archive" }];
    const row = await service().create({ companyId, actor: actor(), agentId, runId, title: "Archive?", body: "Exact set",
      continuationPolicy: "wake_origin_agent", options: [{ id: "archive", label: "Archive", effects: [] }],
      metadata: { kind: "attention_archive_proposal", manifestHash: hashAttentionArchiveManifest(manifest) },
      additionalTargetSnapshots: { [`attention:review:${targetId}`]: { status: "attention", assigneeAgentId: null,
        assigneeUserId: null, updatedAt: activityAt.toISOString(), attentionArchive: manifest[0] } } });
    if (committed) {
      await failSettlement();
      await expect(service().decide({ id: row.id, optionId: "archive", decidedByUserId: userId, userActor: board() }))
        .rejects.toMatchObject({ cause: { message: "settlement unavailable" } });
      await db.execute(sql`drop trigger decision_effect_finish_fail on decisions`);
    } else {
      await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running",
        chosenOptionId: "archive", decidedByUserId: userId, updatedAt: new Date(Date.now() - 120_000),
        metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id)));
    }
    await start(); await settled(row.id, committed ? "failed" : "succeeded"); await quiet();
    const [retained] = await db.select().from(decisionRetention);
    expect(retained.archiveVersion).toBe(1); expect(retained.archivedAt).not.toBeNull();
    expect(wake).toHaveBeenCalledTimes(1);
    if (committed) expect((await service().get(row.id))?.metadata).toHaveProperty("archiveProposalError");
  });

  it("learns another claim written during delivery", async () => {
    const first = await create(), second = await create(); await claim(first.id, new Date(Date.now() - 120_000));
    wake.mockImplementationOnce(async () => { await claim(second.id); });
    await start(); await settled(first.id); await settled(second.id); await quiet();
    expect(await comments()).toHaveLength(2); expect(wake).toHaveBeenCalledTimes(2);
  });

  it("finishes the admitted recovery and leaves later claims for restart during shutdown", async () => {
    const first = await create(), second = await create();
    await claim(first.id, new Date(Date.now() - 180_000)); await claim(second.id, new Date(Date.now() - 120_000));
    let release!: () => void;
    const gate = new Promise<void>(resolve => { release = resolve; });
    wake.mockImplementationOnce(async () => gate);
    const starting = start(); let stopped = false; let stopping: Promise<void>;
    try {
      await vi.waitFor(() => expect(wake).toHaveBeenCalledTimes(1));
      stopping = coordinator!.stop().then(() => { stopped = true; });
      await vi.advanceTimersByTimeAsync(60_000); expect(stopped).toBe(false);
    } finally { release(); }
    await starting; await stopping!; coordinator = undefined;
    expect(wake).toHaveBeenCalledTimes(1); expect(await comments()).toHaveLength(1);
    expect((await service().get(second.id))?.executionStatus).toBe("running");
    await start(); await quiet(); expect(await comments()).toHaveLength(2); expect(wake).toHaveBeenCalledTimes(2);
  });
});