PaperClipAI/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts

import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
  subscribeDatabaseWork,
  costEvents,
  activityLog,
  agents,
  companies,
  completionContracts,
  createDb,
  heartbeatRunEvents,
  heartbeatRuns,
  issueComments,
  issueRelations,
  issues,
  nativeRunFinalizations,
  nativeRunResults,
} from "@paperclipai/db";
import {
  getEmbeddedPostgresTestSupport,
  startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";

const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn() }));
vi.mock("../telemetry.ts", () => ({ getTelemetryClient: () => mockTelemetryClient }));

import { heartbeatService } from "../services/heartbeat.ts";
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { recoveryService } from "../services/recovery/service.ts";

const external = resolveExternalTestDatabaseUrl("PAPERCLIP_STALE_LOCK_TEST_DATABASE_URL");
const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;

if (!embeddedPostgresSupport.supported) {
  console.warn(
    `Skipping embedded Postgres stale-lock sweeper tests on this host: ${
      embeddedPostgresSupport.reason ?? "unsupported environment"
    }`,
  );
}

describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => {
  let db!: ReturnType<typeof createDb>;
  let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;

  beforeAll(async () => {
    if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-stale-lock-sweep-");
    db = createDb(external ?? tempDb!.connectionString);
  }, 20_000);

  afterEach(async () => {
    mockTelemetryClient.track.mockClear();
    await db.delete(nativeRunFinalizations);
    await db.delete(nativeRunResults);
    await db.delete(issueComments);
    await db.delete(issueRelations);
    await db.delete(activityLog);
    await db.delete(heartbeatRunEvents);
    await db.delete(costEvents);
    await db.delete(heartbeatRuns);
    await db.delete(completionContracts);
    await db.delete(issues);
    await db.delete(agents);
    await db.delete(companies);
  });

  afterAll(async () => {
    await db?.$client.end({ timeout: 1 });
    await tempDb?.cleanup();
  });

  async function seed() {
    const companyId = randomUUID();
    const agentId = randomUUID();
    const failedRunId = randomUUID();
    const runningRunId = randomUUID();

    await db.insert(companies).values({
      id: companyId,
      name: "Paperclip",
      issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
      requireBoardApprovalForNewAgents: false,
    });
    await db.insert(agents).values({
      id: agentId,
      companyId,
      name: "Coder",
      role: "engineer",
      status: "active",
      adapterType: "codex_local",
      adapterConfig: {},
      runtimeConfig: {},
      permissions: {},
    });
    await db.insert(heartbeatRuns).values([
      {
        id: failedRunId,
        companyId,
        agentId,
        status: "failed",
        invocationSource: "manual",
        finishedAt: new Date(),
      },
      {
        id: runningRunId,
        companyId,
        agentId,
        status: "running",
        invocationSource: "manual",
        startedAt: new Date(),
      },
    ]);

    return { companyId, agentId, failedRunId, runningRunId };
  }

  it.each([true, false])("preserves explicit retry ownership when finalization wins the sweep write (settler: %s)", async withSettler => {
    const { companyId, agentId, runningRunId, failedRunId } = await seed();
    const issueId = randomUUID();
    await db.insert(issues).values({ id: issueId, companyId, title: "Retry finalization race",
      status: "in_progress", assigneeAgentId: agentId, executionRunId: runningRunId });
    await db.update(heartbeatRuns).set({ processPid: 2_000_000_000,
      contextSnapshot: { issueId, explicitUserContinuation: { previousRunId: failedRunId, commentId: randomUUID() } },
    }).where(eq(heartbeatRuns.id, runningRunId));
    const beforeOrphanedRunTerminalWrite = vi.fn(async () => {
      await db.update(heartbeatRuns).set({ status: "timed_out", finishedAt: new Date() })
        .where(eq(heartbeatRuns.id, runningRunId));
    });
    const settleExplicitContinuationRetry = vi.fn(async () => {});
    const result = await recoveryService(db, { enqueueWakeup: vi.fn(), beforeOrphanedRunTerminalWrite,
      ...(withSettler ? { settleExplicitContinuationRetry } : {}),
    }).sweepStaleIssueLocks();
    expect(beforeOrphanedRunTerminalWrite).toHaveBeenCalledOnce();
    expect(result).toEqual({ cleared: 0, issueIds: [], terminalizedRunIds: [] });
    expect(settleExplicitContinuationRetry).toHaveBeenCalledTimes(withSettler ? 1 : 0);
    if (withSettler) expect(settleExplicitContinuationRetry).toHaveBeenCalledWith(expect.objectContaining({
      id: runningRunId, status: "timed_out",
    }));
    expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]!.executionRunId).toBe(runningRunId);
  });

  it("clears lock columns when checkoutRunId points at a terminal heartbeat run", async () => {
    const { companyId, agentId, failedRunId } = await seed();
    const issueId = randomUUID();
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Stale lock — terminal checkoutRunId",
      // Status off in_progress + checkoutRunId still set → exactly the recurrence shape.
      status: "todo",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: failedRunId,
      executionRunId: null,
    });

    const heartbeat = heartbeatService(db);
    const result = await heartbeat.sweepStaleIssueLocks();

    expect(result.cleared).toBe(1);
    expect(result.issueIds).toEqual([issueId]);

    const row = await db
      .select({
        checkoutRunId: issues.checkoutRunId,
        executionRunId: issues.executionRunId,
        executionLockedAt: issues.executionLockedAt,
      })
      .from(issues)
      .where(eq(issues.id, issueId))
      .then((rows) => rows[0]);
    expect(row).toEqual({ checkoutRunId: null, executionRunId: null, executionLockedAt: null });

    const audit = await db
      .select({ action: activityLog.action, details: activityLog.details })
      .from(activityLog)
      .where(eq(activityLog.action, "issue.stale_lock_cleared"))
      .then((rows) => rows[0]);
    expect(audit?.action).toBe("issue.stale_lock_cleared");
    expect((audit?.details as { clearedCheckoutRunId?: string } | null)?.clearedCheckoutRunId).toBe(
      failedRunId,
    );
  });

  it("does not clear locks while the referenced run is still running", async () => {
    const { companyId, agentId, runningRunId } = await seed();
    const issueId = randomUUID();
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Live lock — must be preserved",
      status: "in_progress",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: runningRunId,
      executionRunId: runningRunId,
      executionLockedAt: new Date(),
    });

    const heartbeat = heartbeatService(db);
    const result = await heartbeat.sweepStaleIssueLocks();

    expect(result.cleared).toBe(0);
    const row = await db
      .select({
        checkoutRunId: issues.checkoutRunId,
        executionRunId: issues.executionRunId,
      })
      .from(issues)
      .where(eq(issues.id, issueId))
      .then((rows) => rows[0]);
    expect(row).toEqual({ checkoutRunId: runningRunId, executionRunId: runningRunId });
  });

  it("does not terminalize a session-goal control run solely because its issue is done", async () => {
    const { companyId, agentId, runningRunId } = await seed();
    const issueId = randomUUID();
    await db
      .update(heartbeatRuns)
      .set({
        contextSnapshot: {
          issueId,
          resumeIntent: true,
          goalControlRequestId: randomUUID(),
          runnerGoalControl: { action: "clear" },
        },
      })
      .where(eq(heartbeatRuns.id, runningRunId));
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Completed goal awaiting clear",
      status: "done",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: runningRunId,
      executionRunId: runningRunId,
      executionLockedAt: new Date(),
    });

    const result = await heartbeatService(db).sweepStaleIssueLocks();

    expect(result.terminalizedRunIds).toEqual([]);
    expect(result.cleared).toBe(0);
    await expect(
      db
        .select({ status: heartbeatRuns.status })
        .from(heartbeatRuns)
        .where(eq(heartbeatRuns.id, runningRunId))
        .then((rows) => rows[0]?.status),
    ).resolves.toBe("running");
  });

  async function seedCompletedOnboardingReport(options: {
    runtimeMode?: "native" | "legacy";
    variant?: "ordinary" | "cancelled" | "wrong_child" | "wrong_reason" | "wrong_agent" | "unfinished_child" | "foreign_child";
  } = {}) {
    const { companyId, agentId, runningRunId } = await seed();
    const issueId = randomUUID();
    const childIssueId = randomUUID();
    const variant = options.variant;
    let childCompanyId = companyId;
    if (variant === "foreign_child") {
      childCompanyId = randomUUID();
      await db.insert(companies).values({
        id: childCompanyId,
        name: "Other company",
        issuePrefix: `T${childCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
      });
    }
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Completed onboarding awaiting its child result",
      originKind: variant === "ordinary" ? "manual" : "onboarding_first_task",
      status: variant === "cancelled" ? "cancelled" : "done",
      assigneeAgentId: variant === "wrong_agent" ? null : agentId,
      checkoutRunId: runningRunId,
      executionRunId: runningRunId,
      executionLockedAt: new Date(),
    });
    await db.insert(issues).values({
      id: childIssueId,
      companyId: childCompanyId,
      title: "Accepted child output",
      status: variant === "unfinished_child" ? "in_progress" : "done",
      parentId: variant === "foreign_child" ? null : issueId,
    });
    await db.update(heartbeatRuns).set({
      runtimeMode: options.runtimeMode ?? "native",
      nativeIssueId: options.runtimeMode === "legacy" ? null : issueId,
      processPid: 2_000_000_000,
      contextSnapshot: {
        issueId,
        wakeReason: variant === "wrong_reason" ? "issue_commented" : "issue_children_completed",
        completedChildIssueId: variant === "wrong_child" ? randomUUID() : childIssueId,
        onboardingCompletion: true,
      },
    }).where(eq(heartbeatRuns.id, runningRunId));
    return { companyId, agentId, issueId, childIssueId, runningRunId };
  }

  it.each(["native", "legacy"] as const)("preserves a live %s completed-onboarding reporting turn during delayed startup", async runtimeMode => {
    const { issueId, runningRunId } = await seedCompletedOnboardingReport({ runtimeMode });
    const result = await recoveryService(db, {
      enqueueWakeup: vi.fn(),
      liveRunExecutions: new Set([runningRunId]),
    }).sweepStaleIssueLocks();

    expect(result).toEqual({ cleared: 0, issueIds: [], terminalizedRunIds: [] });
    expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runningRunId)))[0]?.status).toBe("running");
    expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({
      status: "done", checkoutRunId: runningRunId, executionRunId: runningRunId,
    });
    expect(mockTelemetryClient.track).not.toHaveBeenCalled();
  });

  it.each(["ordinary", "cancelled", "wrong_child", "wrong_reason", "wrong_agent", "unfinished_child", "foreign_child"] as const)("does not exempt an invalid completed-onboarding reporting turn (%s)", async variant => {
    const { issueId, runningRunId } = await seedCompletedOnboardingReport({ variant });
    const result = await recoveryService(db, {
      enqueueWakeup: vi.fn(),
      liveRunExecutions: new Set([runningRunId]),
    }).sweepStaleIssueLocks();

    expect(result.terminalizedRunIds).toEqual([runningRunId]);
    expect(result.cleared).toBe(1);
    expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runningRunId)))[0]?.status).toBe(variant === "cancelled" ? "cancelled" : "succeeded");
    expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ checkoutRunId: null, executionRunId: null });
  });

  it("still interrupts a completed-onboarding reporting turn after its process and execution owner are gone", async () => {
    const { issueId, runningRunId } = await seedCompletedOnboardingReport();
    const result = await recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks();

    expect(result.terminalizedRunIds).toEqual([runningRunId]);
    expect(result.cleared).toBe(1);
    expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runningRunId)))[0]).toMatchObject({
      status: "interrupted", errorCode: "orphaned_running_run",
    });
    expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ checkoutRunId: null, executionRunId: null });
  });

  it("does not clear when checkoutRunId is terminal but executionRunId is still running", async () => {
    const { companyId, agentId, failedRunId, runningRunId } = await seed();
    const issueId = randomUUID();
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Mixed lock — preserve",
      status: "in_progress",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: failedRunId,
      executionRunId: runningRunId,
      executionLockedAt: new Date(),
    });

    const heartbeat = heartbeatService(db);
    const result = await heartbeat.sweepStaleIssueLocks();

    expect(result.cleared).toBe(0);
    const row = await db
      .select({
        checkoutRunId: issues.checkoutRunId,
        executionRunId: issues.executionRunId,
      })
      .from(issues)
      .where(eq(issues.id, issueId))
      .then((rows) => rows[0]);
    expect(row).toEqual({ checkoutRunId: failedRunId, executionRunId: runningRunId });
  });

  it("is idempotent — second pass finds nothing to clear", async () => {
    const { companyId, agentId, failedRunId } = await seed();
    const issueId = randomUUID();
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Idempotency",
      status: "todo",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: failedRunId,
      executionRunId: null,
    });

    const heartbeat = heartbeatService(db);
    const first = await heartbeat.sweepStaleIssueLocks();
    const second = await heartbeat.sweepStaleIssueLocks();
    expect(first.cleared).toBe(1);
    expect(second.cleared).toBe(0);
  });

  it("terminalizes an orphaned running run whose process is gone, then clears the lock", async () => {
    const { companyId, agentId, runningRunId } = await seed();
    // The run recorded a pid, but the process and its sandbox are gone. A pid
    // this large never maps to a live process, so isPidAlive returns false.
    // The issue is not terminal, so only the process-death authority applies.
    await db
      .update(heartbeatRuns)
      .set({ processPid: 2_000_000_000 })
      .where(eq(heartbeatRuns.id, runningRunId));
    const issueId = randomUUID();
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Orphaned running run — terminalize then clear",
      status: "in_progress",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: runningRunId,
      executionRunId: runningRunId,
      executionLockedAt: new Date(),
    });

    const heartbeat = heartbeatService(db);
    const result = await heartbeat.sweepStaleIssueLocks();

    expect(result.terminalizedRunIds).toEqual([runningRunId]);
    expect(result.cleared).toBe(1);

    const run = await db
      .select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })
      .from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId))
      .then((rows) => rows[0]);
    // Process died, outcome unknown, so the backstop uses "interrupted".
    expect(run?.status).toBe("interrupted");
    expect(run?.errorCode).toBe("orphaned_running_run");

    const lock = await db
      .select({ checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId })
      .from(issues)
      .where(eq(issues.id, issueId))
      .then((rows) => rows[0]);
    expect(lock).toEqual({ checkoutRunId: null, executionRunId: null });

    const event = await db
      .select({ message: heartbeatRunEvents.message })
      .from(heartbeatRunEvents)
      .where(eq(heartbeatRunEvents.runId, runningRunId))
      .then((rows) => rows[0]);
    expect(event?.message).toContain("process and sandbox gone");

    // The recovery sweep's own terminal write must emit exactly one
    // agent.task_run event for the run it just terminalized. The write
    // never awaits the emission, so wait for it here instead of asserting
    // it fired synchronously.
    await vi.waitFor(() => {
      expect(mockTelemetryClient.track).toHaveBeenCalledTimes(1);
    });
    expect(mockTelemetryClient.track).toHaveBeenCalledWith(
      "agent.task_run",
      expect.objectContaining({ agent_id: agentId, state: "interrupted" }),
    );
  });

  it.each(["in_progress", "done"])("preserves a live legacy controller lease when the issue is %s", async (status) => {
    const { companyId, agentId, runningRunId } = await seed();
    await db.update(heartbeatRuns).set({
      runtimeMode: "legacy", processPid: 2_000_000_000,
      controllerBootId: randomUUID(),
      controllerLeaseExpiresAt: new Date(Date.now() + 60_000),
    }).where(eq(heartbeatRuns.id, runningRunId));
    const issueId = randomUUID();
    await db.insert(issues).values({
      id: issueId, companyId, title: "Remote controller still owns the run", status,
      assigneeAgentId: agentId, executionRunId: runningRunId, checkoutRunId: runningRunId,
    });
    const result = await recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks();
    expect(result).toEqual({ cleared: 0, issueIds: [], terminalizedRunIds: [] });
    expect(await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "running" }]);
    expect(await db.select({ executionRunId: issues.executionRunId }).from(issues)
      .where(eq(issues.id, issueId))).toEqual([{ executionRunId: runningRunId }]);
  });

  it.each(["renew", "replace", "claim"])("fences a legacy controller %s between the orphan check and terminal write", async (change) => {
    const { companyId, agentId, runningRunId } = await seed();
    const bootId = randomUUID();
    await db.update(heartbeatRuns).set({
      runtimeMode: "legacy", processPid: 2_000_000_000,
      controllerBootId: change === "claim" ? null : bootId,
      controllerLeaseExpiresAt: new Date(Date.now() - 60_000),
    }).where(eq(heartbeatRuns.id, runningRunId));
    await db.insert(issues).values({
      id: randomUUID(), companyId, title: "Controller changed during sweep", status: "in_progress",
      assigneeAgentId: agentId, executionRunId: runningRunId, checkoutRunId: runningRunId,
    });
    const result = await recoveryService(db, {
      enqueueWakeup: vi.fn(),
      beforeOrphanedRunTerminalWrite: async () => {
        await db.update(heartbeatRuns).set({
          controllerBootId: change === "renew" ? bootId : randomUUID(),
          // A replacement invalidates the old snapshot even if its lease expires.
          controllerLeaseExpiresAt: new Date(Date.now() + (change === "replace" ? -30_000 : 60_000)),
        }).where(eq(heartbeatRuns.id, runningRunId));
      },
    }).sweepStaleIssueLocks();
    expect(result).toEqual({ cleared: 0, issueIds: [], terminalizedRunIds: [] });
    expect(await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "running" }]);
    expect(mockTelemetryClient.track).not.toHaveBeenCalled();
  });

  it("preserves a process-less native run while same-run resumption owns its retry", async () => {
    const { companyId, agentId, runningRunId } = await seed();
    const issueId = randomUUID();
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Native same-run retry remains live",
      status: "in_progress",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: runningRunId,
      executionRunId: runningRunId,
      executionLockedAt: new Date(),
    });
    await db
      .update(heartbeatRuns)
      .set({
        runtimeMode: "native",
        nativeIssueId: issueId,
        nativePhase: "retryable_failure",
        processPid: 2_000_000_000,
      })
      .where(eq(heartbeatRuns.id, runningRunId));
    await db.insert(nativeRunFinalizations).values({
      runId: runningRunId,
      companyId,
      issueId,
      phase: "retryable_failure",
      attempt: 1,
      nextAttemptAt: new Date(Date.now() + 30_000),
    });

    const result = await heartbeatService(db).sweepStaleIssueLocks();

    expect(result).toEqual({ cleared: 0, issueIds: [], terminalizedRunIds: [] });
    await expect(db.select({ status: heartbeatRuns.status })
      .from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId)))
      .resolves.toEqual([{ status: "running" }]);
    await expect(db.select({
      checkoutRunId: issues.checkoutRunId,
      executionRunId: issues.executionRunId,
    }).from(issues).where(eq(issues.id, issueId)))
      .resolves.toEqual([{ checkoutRunId: runningRunId, executionRunId: runningRunId }]);
  });

  async function seedPendingNativeResult(phase: string, options: { terminalIssue?: boolean; deferCoordinator?: boolean } = {}) {
    const fixture = await seed();
    const { companyId, agentId, runningRunId } = fixture;
    const issueId = randomUUID();
    const contractId = randomUUID();
    const resultId = randomUUID();
    await db.insert(issues).values({
      id: issueId, companyId, title: "Workspace finalization fixture",
      status: options.terminalIssue ? "done" : "in_progress", assigneeAgentId: agentId,
      executionRunId: runningRunId, checkoutRunId: runningRunId,
    });
    await db.insert(completionContracts).values({
      id: contractId, companyId, issueId, revision: 1,
      schemaVersion: "paperclip.completion-contract.v1", policyVersion: "fixture",
      risk: "standard", completionAuthority: "server_arbiter",
      incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: {},
      canonicalSha256: "fixture-contract", createdByActorType: "system", createdByActorId: "test",
    });
    await db.update(heartbeatRuns).set({
      runtimeMode: "native", nativeIssueId: issueId, nativePhase: phase,
      processPid: 2_000_000_000, completionContractId: contractId,
    }).where(eq(heartbeatRuns.id, runningRunId));
    await db.insert(nativeRunResults).values({
      id: resultId, companyId, issueId, runId: runningRunId, completionContractId: contractId,
      serverFingerprint: "fixture-result", schemaStatus: "accepted", resultJson: {}, canonicalSha256: "fixture-result",
    });
    const recordCoordinator = () => db.insert(nativeRunFinalizations).values({
      runId: runningRunId, companyId, issueId, phase, resultId, attempt: 1,
      nextAttemptAt: new Date(Date.now() + 30_000),
      leaseOwner: "previous-controller", leaseExpiresAt: new Date(Date.now() - 30_000),
    });
    if (!options.deferCoordinator) await recordCoordinator();
    return { ...fixture, issueId, recordCoordinator };
  }

  it.each(["observed", "workspace_finalizing", "ready_for_assessment", "arbitrating", "retryable_failure"])(
    "preserves a completed native result in %s after its provider exits", async phase => {
      const { runningRunId, issueId } = await seedPendingNativeResult(phase);
      const events: string[] = [];
      const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatRunMilestones, event => events.push(event));
      try {
        const recovery = recoveryService(db, { enqueueWakeup: vi.fn() });
        for (let pass = 0; pass < 3; pass++) {
          expect(await recovery.sweepStaleIssueLocks()).toEqual({ cleared: 0, issueIds: [], terminalizedRunIds: [] });
        }
        expect(events).toEqual([]);
      } finally { unsubscribe(); }
      expect(await db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode }).from(heartbeatRuns)
        .where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "running", errorCode: null }]);
      expect(await db.select({ executionRunId: issues.executionRunId }).from(issues)
        .where(eq(issues.id, issueId))).toEqual([{ executionRunId: runningRunId }]);
      expect(mockTelemetryClient.track).not.toHaveBeenCalled();
    },
  );

  it("rechecks native finalization ownership when the coordinator appears after the liveness read", async () => {
    const { runningRunId, recordCoordinator } = await seedPendingNativeResult("retryable_failure", { deferCoordinator: true });
    const beforeOrphanedRunTerminalWrite = vi.fn(async () => { await recordCoordinator(); });
    const result = await recoveryService(db, { enqueueWakeup: vi.fn(), beforeOrphanedRunTerminalWrite }).sweepStaleIssueLocks();
    expect(beforeOrphanedRunTerminalWrite).toHaveBeenCalledOnce();
    expect(result.terminalizedRunIds).toEqual([]);
    expect(await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "running" }]);
  });

  it("still respects terminal issue authority while workspace finalization is pending", async () => {
    const { runningRunId } = await seedPendingNativeResult("retryable_failure", { terminalIssue: true });
    const result = await recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks();
    expect(result.terminalizedRunIds).toContain(runningRunId);
    expect(await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "succeeded" }]);
  });

  it("does not keep an orphan live after native finalization exhausts its retries", async () => {
    const { runningRunId } = await seedPendingNativeResult("terminal_failure");
    const result = await recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks();
    expect(result.terminalizedRunIds).toContain(runningRunId);
    expect(await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "interrupted" }]);
  });

  it("preserves a process-less run while its in-process execution is still finalizing", async () => {
    const { companyId, agentId, runningRunId } = await seed();
    const issueId = randomUUID();
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Native finalization remains live",
      status: "in_progress",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: runningRunId,
      executionRunId: runningRunId,
      executionLockedAt: new Date(),
    });
    await db
      .update(heartbeatRuns)
      .set({
        runtimeMode: "native",
        processPid: 2_000_000_000,
      })
      .where(eq(heartbeatRuns.id, runningRunId));

    const result = await recoveryService(db, {
      enqueueWakeup: vi.fn(),
      liveRunExecutions: new Set([runningRunId]),
    }).sweepStaleIssueLocks();

    expect(result).toEqual({
      cleared: 0,
      issueIds: [],
      terminalizedRunIds: [],
    });
    await expect(db.select({ status: heartbeatRuns.status })
      .from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId)))
      .resolves.toEqual([{ status: "running" }]);
    await expect(db.select({
      checkoutRunId: issues.checkoutRunId,
      executionRunId: issues.executionRunId,
    }).from(issues).where(eq(issues.id, issueId)))
      .resolves.toEqual([{ checkoutRunId: runningRunId, executionRunId: runningRunId }]);
  });

  it("terminalizes a running run whose issue is terminal, even while the process stays alive (reuse-lease path)", async () => {
    // Reuse Lease ON stops the sandbox but keeps the server process alive, so
    // the in-memory handle and the recorded pid can both persist. The
    // process-death authority misses this case. The issue-terminal authority
    // catches it: the issue reached "done" while the run row stayed "running".
    const { companyId, agentId, runningRunId } = await seed();
    // process.pid is the live test process, so isPidAlive returns true.
    await db
      .update(heartbeatRuns)
      .set({ processPid: process.pid })
      .where(eq(heartbeatRuns.id, runningRunId));
    const issueId = randomUUID();
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Reused sandbox stopped — issue done, run still running",
      status: "done",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: runningRunId,
      executionRunId: runningRunId,
      executionLockedAt: new Date(),
    });

    const heartbeat = heartbeatService(db);
    const result = await heartbeat.sweepStaleIssueLocks();

    expect(result.terminalizedRunIds).toEqual([runningRunId]);
    expect(result.cleared).toBe(1);

    const run = await db
      .select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })
      .from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId))
      .then((rows) => rows[0]);
    // The issue is "done", so the terminal run status is "succeeded". A
    // succeeded run carries no error code.
    expect(run?.status).toBe("succeeded");
    expect(run?.errorCode).toBeNull();

    const lock = await db
      .select({ checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId })
      .from(issues)
      .where(eq(issues.id, issueId))
      .then((rows) => rows[0]);
    expect(lock).toEqual({ checkoutRunId: null, executionRunId: null });

    const event = await db
      .select({ message: heartbeatRunEvents.message })
      .from(heartbeatRunEvents)
      .where(eq(heartbeatRunEvents.runId, runningRunId))
      .then((rows) => rows[0]);
    expect(event?.message).toContain("issue reached a terminal status");

    // The terminal write never awaits the telemetry emission, so wait for
    // it here instead of asserting it fired synchronously.
    await vi.waitFor(() => {
      expect(mockTelemetryClient.track).toHaveBeenCalledTimes(1);
    });
    expect(mockTelemetryClient.track).toHaveBeenCalledWith(
      "agent.task_run",
      expect.objectContaining({ agent_id: agentId, state: "succeeded" }),
    );
  });

  it("terminalizes a running run to cancelled when its issue is cancelled (reuse-lease path)", async () => {
    const { companyId, agentId, runningRunId } = await seed();
    await db
      .update(heartbeatRuns)
      .set({ processPid: process.pid })
      .where(eq(heartbeatRuns.id, runningRunId));
    const issueId = randomUUID();
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Reused sandbox stopped — issue cancelled, run still running",
      status: "cancelled",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: runningRunId,
      executionRunId: runningRunId,
      executionLockedAt: new Date(),
    });

    const heartbeat = heartbeatService(db);
    const result = await heartbeat.sweepStaleIssueLocks();

    expect(result.terminalizedRunIds).toEqual([runningRunId]);

    const runStatus = await db
      .select({ status: heartbeatRuns.status })
      .from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId))
      .then((rows) => rows[0]?.status);
    expect(runStatus).toBe("cancelled");

    // The terminal write never awaits the telemetry emission, so wait for
    // it here instead of asserting it fired synchronously.
    await vi.waitFor(() => {
      expect(mockTelemetryClient.track).toHaveBeenCalledTimes(1);
    });
    expect(mockTelemetryClient.track).toHaveBeenCalledWith(
      "agent.task_run",
      expect.objectContaining({ agent_id: agentId, state: "cancelled" }),
    );
  });

  it("does not terminalize a running run whose process is alive and whose issue is not terminal", async () => {
    const { companyId, agentId, runningRunId } = await seed();
    // process.pid is the live test process, so isPidAlive returns true.
    await db
      .update(heartbeatRuns)
      .set({ processPid: process.pid })
      .where(eq(heartbeatRuns.id, runningRunId));
    const issueId = randomUUID();
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Live run — preserve",
      status: "in_progress",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: runningRunId,
      executionRunId: runningRunId,
      executionLockedAt: new Date(),
    });

    const heartbeat = heartbeatService(db);
    const result = await heartbeat.sweepStaleIssueLocks();

    expect(result.terminalizedRunIds).toEqual([]);
    expect(result.cleared).toBe(0);

    const runStatus = await db
      .select({ status: heartbeatRuns.status })
      .from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId))
      .then((rows) => rows[0]?.status);
    expect(runStatus).toBe("running");

    // No terminal write happened, so no telemetry event fires.
    expect(mockTelemetryClient.track).not.toHaveBeenCalled();
  });

  it("does not terminalize or clear issue locks when an ownership hold arrives after the sweep snapshot", async () => {
    const { companyId, agentId, runningRunId } = await seed();
    const issueId = randomUUID();
    await db
      .update(heartbeatRuns)
      .set({
        runtimeMode: "native",
        nativeIssueId: issueId,
        processPid: process.pid,
      })
      .where(eq(heartbeatRuns.id, runningRunId));
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Ownership hold races stale issue-lock recovery",
      status: "done",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: runningRunId,
      executionRunId: runningRunId,
      executionLockedAt: new Date(),
    });

    let releaseTerminalWrite!: () => void;
    const terminalWriteReleased = new Promise<void>((resolve) => {
      releaseTerminalWrite = resolve;
    });
    let terminalWriteReached!: () => void;
    const atTerminalWrite = new Promise<void>((resolve) => {
      terminalWriteReached = resolve;
    });
    const sweep = recoveryService(db, {
      enqueueWakeup: vi.fn(),
      beforeOrphanedRunTerminalWrite: async (runId) => {
        if (runId !== runningRunId) return;
        terminalWriteReached();
        await terminalWriteReleased;
      },
    }).sweepStaleIssueLocks();

    try {
      await atTerminalWrite;
      await db
        .update(heartbeatRuns)
        .set({
          nativePhase: "terminal_failure",
          errorCode: "native_execution_ownership_unverified",
        })
        .where(eq(heartbeatRuns.id, runningRunId));
    } finally {
      releaseTerminalWrite();
    }

    await expect(sweep).resolves.toEqual({
      cleared: 0,
      issueIds: [],
      terminalizedRunIds: [],
    });
    await expect(
      db
        .select({
          status: heartbeatRuns.status,
          nativePhase: heartbeatRuns.nativePhase,
          errorCode: heartbeatRuns.errorCode,
        })
        .from(heartbeatRuns)
        .where(eq(heartbeatRuns.id, runningRunId)),
    ).resolves.toEqual([
      {
        status: "running",
        nativePhase: "terminal_failure",
        errorCode: "native_execution_ownership_unverified",
      },
    ]);
    await expect(
      db
        .select({
          checkoutRunId: issues.checkoutRunId,
          executionRunId: issues.executionRunId,
        })
        .from(issues)
        .where(eq(issues.id, issueId)),
    ).resolves.toEqual([
      {
        checkoutRunId: runningRunId,
        executionRunId: runningRunId,
      },
    ]);
    expect(mockTelemetryClient.track).not.toHaveBeenCalled();
  });

  it("does not terminalize a live run that a terminal issue and an active issue both reference", async () => {
    // A stale lock on a terminal issue and the real lock on an active issue can
    // point at the same running run. The terminal reference alone must not
    // terminalize the run, because the run is still live for the active issue.
    const { companyId, agentId, runningRunId } = await seed();
    // process.pid is the live test process, so isPidAlive returns true.
    await db
      .update(heartbeatRuns)
      .set({ processPid: process.pid })
      .where(eq(heartbeatRuns.id, runningRunId));

    const terminalIssueId = randomUUID();
    const activeIssueId = randomUUID();
    await db.insert(issues).values([
      {
        id: terminalIssueId,
        companyId,
        title: "Terminal issue holds a stale lock on the shared run",
        status: "done",
        priority: "high",
        assigneeAgentId: agentId,
        checkoutRunId: runningRunId,
        executionRunId: null,
      },
      {
        id: activeIssueId,
        companyId,
        title: "Active issue owns the live shared run",
        status: "in_progress",
        priority: "high",
        assigneeAgentId: agentId,
        checkoutRunId: runningRunId,
        executionRunId: runningRunId,
        executionLockedAt: new Date(),
      },
    ]);

    const heartbeat = heartbeatService(db);
    const result = await heartbeat.sweepStaleIssueLocks();

    // The run stays live, so the sweep terminalizes nothing and clears nothing.
    expect(result.terminalizedRunIds).toEqual([]);
    expect(result.cleared).toBe(0);

    const runStatus = await db
      .select({ status: heartbeatRuns.status })
      .from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId))
      .then((rows) => rows[0]?.status);
    expect(runStatus).toBe("running");

    const activeLock = await db
      .select({ checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId })
      .from(issues)
      .where(eq(issues.id, activeIssueId))
      .then((rows) => rows[0]);
    expect(activeLock).toEqual({ checkoutRunId: runningRunId, executionRunId: runningRunId });
  });

  it("does not terminalize a shared live run when its context snapshot names the terminal issue", async () => {
    // The run context snapshot names the terminal issue. The context-snapshot
    // fallback in terminalizeOrphanedRunningRun could read that terminal status
    // and terminalize the run. An active issue still owns the run, so the sweep
    // must suppress the fallback and keep the run live.
    const { companyId, agentId, runningRunId } = await seed();
    // process.pid is the live test process, so isPidAlive returns true.
    await db
      .update(heartbeatRuns)
      .set({ processPid: process.pid })
      .where(eq(heartbeatRuns.id, runningRunId));

    const terminalIssueId = randomUUID();
    const activeIssueId = randomUUID();
    // The run context snapshot names the terminal issue. This is the path the
    // shared-run guard must still block.
    await db
      .update(heartbeatRuns)
      .set({ contextSnapshot: { issueId: terminalIssueId } })
      .where(eq(heartbeatRuns.id, runningRunId));
    await db.insert(issues).values([
      {
        id: terminalIssueId,
        companyId,
        title: "Terminal issue named in the run context snapshot",
        status: "done",
        priority: "high",
        assigneeAgentId: agentId,
        checkoutRunId: runningRunId,
        executionRunId: null,
      },
      {
        id: activeIssueId,
        companyId,
        title: "Active issue owns the live shared run",
        status: "in_progress",
        priority: "high",
        assigneeAgentId: agentId,
        checkoutRunId: runningRunId,
        executionRunId: runningRunId,
        executionLockedAt: new Date(),
      },
    ]);

    const heartbeat = heartbeatService(db);
    const result = await heartbeat.sweepStaleIssueLocks();

    // The run stays live, so the sweep terminalizes nothing and clears nothing.
    expect(result.terminalizedRunIds).toEqual([]);
    expect(result.cleared).toBe(0);

    const runStatus = await db
      .select({ status: heartbeatRuns.status })
      .from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId))
      .then((rows) => rows[0]?.status);
    expect(runStatus).toBe("running");

    const activeLock = await db
      .select({ checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId })
      .from(issues)
      .where(eq(issues.id, activeIssueId))
      .then((rows) => rows[0]);
    expect(activeLock).toEqual({ checkoutRunId: runningRunId, executionRunId: runningRunId });
  });

  it("still clears the lock when the audit write fails after terminalization", async () => {
    const { companyId, agentId, runningRunId } = await seed();
    // The run recorded a pid that never maps to a live process, so the sweep
    // decides to terminalize it. The issue is not terminal, so the
    // process-death authority drives the terminalization here.
    await db
      .update(heartbeatRuns)
      .set({ processPid: 2_000_000_000 })
      .where(eq(heartbeatRuns.id, runningRunId));
    const issueId = randomUUID();
    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Audit write fails — still clear the lock",
      status: "in_progress",
      priority: "high",
      assigneeAgentId: agentId,
      checkoutRunId: runningRunId,
      executionRunId: runningRunId,
      executionLockedAt: new Date(),
    });

    // Make only the audit-event insert fail. The run update commits the
    // terminal status first, so the audit write is best-effort. The sweep must
    // catch the failure and still clear the lock.
    await db.execute(sql`
      create function reject_stale_lock_audit() returns trigger language plpgsql as $$
      begin raise exception 'simulated audit write failure'; end;
      $$
    `);

    try {
      await db.execute(sql`
        create trigger reject_stale_lock_audit before insert on heartbeat_run_events
        for each row execute function reject_stale_lock_audit()
      `);
      const heartbeat = heartbeatService(db);
      const result = await heartbeat.sweepStaleIssueLocks();

      expect(result.terminalizedRunIds).toEqual([runningRunId]);
      expect(result.cleared).toBe(1);
    } finally {
      await db.execute(sql`drop trigger if exists reject_stale_lock_audit on heartbeat_run_events`);
      await db.execute(sql`drop function reject_stale_lock_audit()`);
    }

    // The run reached its terminal status even though the audit write failed.
    const runStatus = await db
      .select({ status: heartbeatRuns.status, nextEventSeq: heartbeatRuns.nextEventSeq })
      .from(heartbeatRuns)
      .where(eq(heartbeatRuns.id, runningRunId))
      .then((rows) => rows[0]);
    expect(runStatus).toEqual({ status: "interrupted", nextEventSeq: 1 });

    // The sweep cleared the lock in the same pass.
    const lock = await db
      .select({ checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId })
      .from(issues)
      .where(eq(issues.id, issueId))
      .then((rows) => rows[0]);
    expect(lock).toEqual({ checkoutRunId: null, executionRunId: null });

    // The audit write failed, so no run event exists for this run.
    const events = await db
      .select({ id: heartbeatRunEvents.id })
      .from(heartbeatRunEvents)
      .where(eq(heartbeatRunEvents.runId, runningRunId));
    expect(events).toEqual([]);
  });
});