PaperClipAI/server/src/__tests__/heartbeat-workspace-busy.test.ts

import { enableRunFailureReporting } from "./helpers/run-failure-reporting.js";
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { and, eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
  costEvents,
  agents,
  authUsers,
  agentRuntimeState,
  agentWakeupRequests,
  activityLog,
  budgetPolicies,
  companies,
  companyMemberships,
  companySkills,
  createDb,
  closeRegisteredClients,
  environments,
  environmentLeases,
  executionWorkspaces,
  heartbeatRunEvents,
  heartbeatRuns,
  issueComments,
  issueRelations,
  issues,
  projects,
  projectWorkspaces,
} from "@paperclipai/db";
import {
  getEmbeddedPostgresTestSupport,
  startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js";
import {
  registerServerAdapter,
  unregisterServerAdapter,
  type AdapterExecutionContext,
} from "../adapters/index.ts";
import {
  WORKSPACE_BUSY_ERROR_CODE,
  WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS,
  WORKSPACE_BUSY_RETRY_BASE_DELAY_MS,
  WORKSPACE_BUSY_RETRY_JITTER_MS,
  WORKSPACE_BUSY_RETRY_REASON,
  WORKSPACE_BUSY_RETRY_WAKE_REASON,
  computeWorkspaceBusyRetryDelayMs,
  heartbeatService,
} from "../services/heartbeat.ts";
import { instanceSettingsService } from "../services/instance-settings.ts";
import { reportRunFailure, waitForPendingRunFailureReports } from "../services/run-failure-report.js";

const captureRunFailure = vi.hoisted(() => vi.fn());
vi.mock("../sentry.js", async (importOriginal) => ({
  ...await importOriginal<typeof import("../sentry.js")>(),
  captureRunFailure,
}));

// Exercise the real SSH lease and heartbeat paths without connecting to a host.
vi.mock("@paperclipai/adapter-utils/ssh", async (importOriginal) => ({
  ...await importOriginal<typeof import("@paperclipai/adapter-utils/ssh")>(),
  ensureSshWorkspaceReady: async (config: { remoteWorkspacePath: string }) => ({
    remoteCwd: config.remoteWorkspacePath,
  }),
}));

const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();
const embeddedPostgresSupport = externalTestDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
const WORKSPACE_BUSY_TEST_ADAPTER = "workspace_busy_test";

if (!embeddedPostgresSupport.supported) {
  console.warn(
    `Skipping embedded Postgres workspace-busy tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
  );
}

describe("computeWorkspaceBusyRetryDelayMs", () => {
  it("stays within the base-delay-to-base-plus-jitter window", () => {
    expect(computeWorkspaceBusyRetryDelayMs(() => 0)).toBe(WORKSPACE_BUSY_RETRY_BASE_DELAY_MS);
    expect(computeWorkspaceBusyRetryDelayMs(() => 0.5)).toBe(
      WORKSPACE_BUSY_RETRY_BASE_DELAY_MS + WORKSPACE_BUSY_RETRY_JITTER_MS / 2,
    );
    expect(computeWorkspaceBusyRetryDelayMs(() => 1)).toBe(
      WORKSPACE_BUSY_RETRY_BASE_DELAY_MS + WORKSPACE_BUSY_RETRY_JITTER_MS,
    );
  });

  it("clamps out-of-range random sources instead of leaving the window", () => {
    expect(computeWorkspaceBusyRetryDelayMs(() => -5)).toBe(WORKSPACE_BUSY_RETRY_BASE_DELAY_MS);
    expect(computeWorkspaceBusyRetryDelayMs(() => 7)).toBe(
      WORKSPACE_BUSY_RETRY_BASE_DELAY_MS + WORKSPACE_BUSY_RETRY_JITTER_MS,
    );
  });
});

describeEmbeddedPostgres("shared-workspace run serialization", () => {
  let db!: ReturnType<typeof createDb>;
  let heartbeat!: ReturnType<typeof heartbeatService>;
  let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
  let workspaceCwd!: string;
  const executedRunIds: string[] = [];
  const executedInputs = new Map<string, AdapterExecutionContext>();
  const failExecutionRunIds = new Set<string>();

  beforeAll(async () => {
    if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl);
    else {
      tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-workspace-busy-");
      db = createDb(tempDb.connectionString);
    }
    heartbeat = heartbeatService(db);
    workspaceCwd = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-busy-"));
    registerServerAdapter({
      type: WORKSPACE_BUSY_TEST_ADAPTER,
      execute: async (input) => {
        executedRunIds.push(input.runId);
        executedInputs.set(input.runId, input);
        if (failExecutionRunIds.has(input.runId)) {
          return {
            exitCode: 1,
            signal: null,
            timedOut: false,
            errorMessage: "Synthetic unexpected adapter failure after workspace wait",
            resultJson: {},
          };
        }
        return {
          exitCode: 0,
          signal: null,
          timedOut: false,
          resultJson: {},
        };
      },
      testEnvironment: async () => ({
        adapterType: WORKSPACE_BUSY_TEST_ADAPTER,
        status: "pass",
        checks: [],
        testedAt: new Date().toISOString(),
      }),
    });
  }, 20_000);

  afterEach(async () => {
    // Seeded holder runs are synthetic "running" rows with no real execution
    // behind them; cancel them first so the drain helper does not spin
    // waiting for them to finish.
    await db
      .update(heartbeatRuns)
      .set({ status: "cancelled", finishedAt: new Date() })
      .where(eq(heartbeatRuns.status, "running"));
    await drainHeartbeatRunsToQuiescence(db, heartbeat);
    await waitForPendingRunFailureReports();
    await cleanupFixture();
    executedRunIds.length = 0;
    executedInputs.clear();
    failExecutionRunIds.clear();
    captureRunFailure.mockClear();
    await instanceSettingsService(db).updateGeneral({ executionMode: "any" });
    await instanceSettingsService(db).updateExperimental({ enableManagedSandboxOnly: false });
    vi.unstubAllEnvs();
  });

  afterAll(async () => {
    unregisterServerAdapter(WORKSPACE_BUSY_TEST_ADAPTER);
    if (workspaceCwd) await fs.rm(workspaceCwd, { recursive: true, force: true });
    if (externalTestDatabaseUrl) await closeRegisteredClients(externalTestDatabaseUrl);
    await db.$client.end();
    await tempDb?.cleanup();
  });

  async function cleanupFixture() {
    for (let attempt = 0; attempt < 5; attempt += 1) {
      try {
        await cleanupFixtureOnce();
        return;
      } catch (error) {
        if (attempt === 4) throw error;
        await new Promise((resolve) => setTimeout(resolve, 50));
      }
    }
  }

  async function cleanupFixtureOnce() {
    await db.delete(costEvents);
    await db.delete(activityLog);
    await db.delete(environmentLeases);
    await db.delete(issueComments);
    await db.delete(issueRelations);
    await db.delete(issues);
    await db.delete(executionWorkspaces);
    await db.delete(projectWorkspaces);
    await db.delete(projects);
    await db.delete(heartbeatRunEvents);
    await db.delete(activityLog);
    await new Promise((resolve) => setTimeout(resolve, 25));
    await db.delete(heartbeatRunEvents);
    await db.delete(activityLog);
    await db.delete(heartbeatRuns);
    await db.delete(agentWakeupRequests);
    await db.delete(agentRuntimeState);
    await db.delete(budgetPolicies);
    await db.delete(agents);
    await db.delete(environments);
    await db.delete(companySkills);
    await db.delete(companyMemberships);
    await db.delete(companies);
  }

  async function waitForRunToLeaveActiveStates(runId: string, timeoutMs = 10_000) {
    const deadline = Date.now() + timeoutMs;
    while (Date.now() < deadline) {
      const run = await heartbeat.getRun(runId);
      if (run && !["queued", "running"].includes(run.status)) return run;
      await new Promise((resolve) => setTimeout(resolve, 50));
    }
    return await heartbeat.getRun(runId);
  }

  // A deferral does two writes in order: first it cancels the original run,
  // then it inserts the scheduled-retry row. waitForRunToLeaveActiveStates
  // returns after the first write, so a read of the retry row can land before
  // the second write and find nothing. Poll until the retry row exists so the
  // retry-row assertions never observe the gap between the two writes.
  async function waitForRetryRun(originalRunId: string, timeoutMs = 10_000) {
    const deadline = Date.now() + timeoutMs;
    while (Date.now() < deadline) {
      const retryRun = await db
        .select()
        .from(heartbeatRuns)
        .where(eq(heartbeatRuns.retryOfRunId, originalRunId))
        .then((rows) => rows[0] ?? null);
      if (retryRun) return retryRun;
      await new Promise((resolve) => setTimeout(resolve, 50));
    }
    return await db
      .select()
      .from(heartbeatRuns)
      .where(eq(heartbeatRuns.retryOfRunId, originalRunId))
      .then((rows) => rows[0] ?? null);
  }

  interface WorkspaceFixture {
    companyId: string;
    projectId: string;
    projectWorkspaceId: string;
    holderAgentId: string;
    holderIssueId: string;
    holderRunId: string;
    agentId: string;
    issueId: string;
    nonAssigneeAgentId: string;
    sharedExecutionWorkspaceId: string | null;
  }

  async function seedWorkspaceFixture(input?: {
    // Bind both issues to one reused shared execution workspace, as when a child
    // task inherits the workspace of a parent whose run is still live.
    sharedExecutionWorkspace?: boolean;
    holderIssueWorkspaceSettings?: Record<string, unknown> | null;
    holderProjectWorkspaceId?: string;
    holderActivityAt?: Date;
    issueWorkspaceSettings?: Record<string, unknown> | null;
    projectWorkspacePolicy?: Record<string, unknown>;
    agentEnvironmentDriver?: "local" | "ssh" | "sandbox";
  }): Promise<WorkspaceFixture> {
    await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true });
    const companyId = randomUUID();
    const projectId = randomUUID();
    const projectWorkspaceId = randomUUID();
    const holderAgentId = randomUUID();
    const holderIssueId = randomUUID();
    const holderRunId = randomUUID();
    const agentId = randomUUID();
    const issueId = randomUUID();
    const nonAssigneeAgentId = randomUUID();
    const sharedExecutionWorkspaceId = input?.sharedExecutionWorkspace ? randomUUID() : null;
    // Deferral/retry tests exercise sandbox protection. Local and SSH folders
    // must remain concurrent even when an older policy requests serialization.
    const agentEnvironmentDriver = input?.agentEnvironmentDriver ?? "sandbox";
    const agentEnvironmentId = agentEnvironmentDriver === "local" ? null : randomUUID();
    const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
    const now = new Date();

    await db.insert(companies).values({
      id: companyId,
      name: "Paperclip",
      issuePrefix,
      requireBoardApprovalForNewAgents: false,
      defaultResponsibleUserId: "responsible-user",
    });
    await db.insert(authUsers).values({
      id: "responsible-user", name: "Responsible user", email: "workspace-busy@example.test",
      createdAt: now, updatedAt: now,
    }).onConflictDoNothing();
    await db.insert(companyMemberships).values({
      companyId, principalType: "user", principalId: "responsible-user",
      status: "active", membershipRole: "member",
    });

    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: "Workspace Busy Project",
      executionWorkspacePolicy: input?.projectWorkspacePolicy ?? null,
    });

    await db.insert(projectWorkspaces).values({
      id: projectWorkspaceId,
      companyId,
      projectId,
      name: "Primary workspace",
      sourceType: "local_path",
      cwd: workspaceCwd,
      isPrimary: true,
    });

    if (agentEnvironmentId) {
      await db.insert(environments).values({
        id: agentEnvironmentId,
        companyId,
        name: `Workspace busy ${agentEnvironmentDriver} ${agentEnvironmentId}`,
        driver: agentEnvironmentDriver,
        status: "active",
        config: agentEnvironmentDriver === "ssh"
          ? { host: "workspace-test.invalid", username: "test", remoteWorkspacePath: workspaceCwd }
          : { provider: "fake", image: "fake:test", reuseLease: false },
      });
    }

    const holderProjectWorkspaceId = input?.holderProjectWorkspaceId ?? projectWorkspaceId;
    if (holderProjectWorkspaceId !== projectWorkspaceId) {
      await db.insert(projectWorkspaces).values({
        id: holderProjectWorkspaceId,
        companyId,
        projectId,
        name: "Secondary workspace",
        sourceType: "local_path",
        cwd: workspaceCwd,
      });
    }

    for (const [id, name] of [
      [holderAgentId, "HolderCoder"],
      [agentId, "DeferredCoder"],
      [nonAssigneeAgentId, "CommenterCoder"],
    ] as const) {
      await db.insert(agents).values({
        id,
        companyId,
        name,
        role: "engineer",
        status: id === holderAgentId ? "running" : "idle",
        adapterType: WORKSPACE_BUSY_TEST_ADAPTER,
        adapterConfig: {},
        runtimeConfig: {
          heartbeat: {
            wakeOnDemand: true,
            maxConcurrentRuns: 1,
          },
        },
        ...(agentEnvironmentId ? { defaultEnvironmentId: agentEnvironmentId } : {}),
        permissions: {},
      });
    }

    if (sharedExecutionWorkspaceId) {
      await db.insert(executionWorkspaces).values({
        id: sharedExecutionWorkspaceId,
        companyId,
        projectId,
        mode: "shared_workspace",
        strategyType: "project_primary",
        name: "Inherited parent workspace",
        status: "active",
        cwd: workspaceCwd,
        providerType: "local_fs",
        providerRef: workspaceCwd,
      });
    }

    const holderActivityAt = input?.holderActivityAt ?? now;
    await db.insert(heartbeatRuns).values({
      id: holderRunId,
      companyId,
      agentId: holderAgentId,
      invocationSource: "assignment",
      triggerDetail: "system",
      status: "running",
      startedAt: holderActivityAt,
      lastOutputAt: holderActivityAt,
      contextSnapshot: {
        issueId: holderIssueId,
        wakeReason: "issue_assigned",
      },
      createdAt: holderActivityAt,
      updatedAt: holderActivityAt,
    });

    await db.insert(issues).values({
      id: holderIssueId,
      companyId,
      title: "Holder issue",
      status: "in_progress",
      priority: "medium",
      responsibleUserId: "responsible-user",
      assigneeAgentId: holderAgentId,
      projectId,
      projectWorkspaceId: holderProjectWorkspaceId,
      executionRunId: holderRunId,
      executionAgentNameKey: "holdercoder",
      executionLockedAt: now,
      issueNumber: 1,
      identifier: `${issuePrefix}-1`,
      ...(sharedExecutionWorkspaceId ? { executionWorkspaceId: sharedExecutionWorkspaceId } : {}),
      ...(input?.holderIssueWorkspaceSettings !== undefined
        ? { executionWorkspaceSettings: input.holderIssueWorkspaceSettings }
        : {}),
    });

    await db.insert(issues).values({
      id: issueId,
      companyId,
      title: "Deferred issue",
      status: "in_progress",
      priority: "medium",
      responsibleUserId: "responsible-user",
      assigneeAgentId: agentId,
      projectId,
      projectWorkspaceId,
      issueNumber: 2,
      identifier: `${issuePrefix}-2`,
      ...(sharedExecutionWorkspaceId
        ? { executionWorkspaceId: sharedExecutionWorkspaceId, executionWorkspacePreference: "reuse_existing" }
        : {}),
      executionWorkspaceSettings:
        input?.issueWorkspaceSettings === undefined
          ? { sharedWorkspaceConcurrency: "serialize" }
          : input.issueWorkspaceSettings,
    });

    return {
      companyId,
      projectId,
      projectWorkspaceId,
      holderAgentId,
      holderIssueId,
      holderRunId,
      agentId,
      issueId,
      nonAssigneeAgentId,
      sharedExecutionWorkspaceId,
    };
  }

  it("auto dispatches alongside a local holder and adds coordination context", async () => {
    const fixture = await seedWorkspaceFixture({
      issueWorkspaceSettings: { sharedWorkspaceConcurrency: "auto" },
      agentEnvironmentDriver: "local",
    });

    const run = await heartbeat.invoke(
      fixture.agentId,
      "assignment",
      { issueId: fixture.issueId, wakeReason: "issue_assigned" },
      "system",
    );
    expect(run).not.toBeNull();

    const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
    expect(finishedRun?.status).toBe("succeeded");
    expect(executedRunIds).toContain(run!.id);
    expect(executedInputs.get(run!.id)?.context.paperclipTaskMarkdown).toContain(
      `shared workspace is concurrently held by run ${fixture.holderRunId}`,
    );
    expect(executedInputs.get(run!.id)?.context.paperclipTaskMarkdown).toContain(
      "expect concurrent mutations, coordinate via commits",
    );
  });

  it("auto defers when the final environment driver is sandbox", async () => {
    const fixture = await seedWorkspaceFixture({
      issueWorkspaceSettings: { sharedWorkspaceConcurrency: "auto" },
      agentEnvironmentDriver: "sandbox",
    });

    const run = await heartbeat.invoke(
      fixture.agentId,
      "assignment",
      { issueId: fixture.issueId, wakeReason: "issue_assigned" },
      "system",
    );
    expect(run).not.toBeNull();

    const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
    expect(finishedRun?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE);
    expect(executedRunIds).not.toContain(run!.id);
  });

  it("auto defers when instance policy forces Kubernetes", async () => {
    const fixture = await seedWorkspaceFixture({
      issueWorkspaceSettings: { sharedWorkspaceConcurrency: "auto" },
      agentEnvironmentDriver: "local",
    });
    await db.insert(environments).values({
      id: randomUUID(),
      companyId: fixture.companyId,
      name: `Managed Kubernetes ${fixture.companyId}`,
      driver: "sandbox",
      status: "active",
      config: { provider: "kubernetes" },
      metadata: { managedKubernetesSandbox: true },
    });
    await instanceSettingsService(db).updateGeneral({ executionMode: "kubernetes" });

    const run = await heartbeat.invoke(
      fixture.agentId,
      "assignment",
      { issueId: fixture.issueId, wakeReason: "issue_assigned" },
      "system",
    );
    expect(run).not.toBeNull();

    const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
    expect(finishedRun?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE);
    expect(executedRunIds).not.toContain(run!.id);
  });

  it.each(["any", "invalid"])("refuses local fallback when Kubernetes is required but bootstrap mode is %s", async (mode) => {
    const fixture = await seedWorkspaceFixture({ agentEnvironmentDriver: "local" });
    vi.stubEnv("PAPERCLIP_EXECUTION_MODE", mode);
    await instanceSettingsService(db).updateGeneral({ executionMode: "kubernetes" });

    const run = await heartbeat.invoke(fixture.agentId, "assignment", { issueId: fixture.issueId }, "system");
    expect(run).not.toBeNull();
    await heartbeat.drainActiveRunExecutions();

    const finished = await heartbeat.getRun(run!.id);
    expect(finished?.status).toBe("failed");
    expect(finished?.error).toContain("no managed Kubernetes environment is configured");
    expect(executedRunIds).not.toContain(run!.id);
    expect(await db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, run!.id))).toHaveLength(0);
    expect((await heartbeat.getRun(fixture.holderRunId))?.status).toBe("running");
  });

  it("lazily provisions the required Kubernetes environment before applying workspace contention", async () => {
    const fixture = await seedWorkspaceFixture({ agentEnvironmentDriver: "local" });
    for (const key of Object.keys(process.env).filter(key => key.startsWith("PAPERCLIP_K8S_"))) {
      vi.stubEnv(key, undefined);
    }
    vi.stubEnv("PAPERCLIP_EXECUTION_MODE", "kubernetes");
    vi.stubEnv("PAPERCLIP_K8S_IN_CLUSTER", "true");
    vi.stubEnv("PAPERCLIP_K8S_BACKEND", "job");
    await instanceSettingsService(db).updateGeneral({ executionMode: "kubernetes" });

    const run = await heartbeat.invoke(fixture.agentId, "assignment", { issueId: fixture.issueId }, "system");
    expect(run).not.toBeNull();
    await heartbeat.drainActiveRunExecutions();

    const configured = await db.select().from(environments).where(eq(environments.driver, "sandbox"));
    expect(configured).toMatchObject([{
      status: "active",
      config: { provider: "kubernetes", inCluster: true, backend: "job" },
      metadata: { managedKubernetesSandbox: true },
    }]);
    expect((await heartbeat.getRun(run!.id))?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE);
    expect(executedRunIds).not.toContain(run!.id);
    expect(await db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, run!.id))).toHaveLength(0);
    expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run!.id))).toHaveLength(1);
  });

  it.each(["missing", "archived"])("refuses local execution when the managed sandbox is %s", async (state) => {
    const fixture = await seedWorkspaceFixture({ agentEnvironmentDriver: "local" });
    if (state === "archived") {
      await db.insert(environments).values({
        name: "Unavailable managed sandbox", driver: "sandbox", status: "archived",
        config: { provider: "fake" }, metadata: { managedByPaperclip: true },
      });
    }
    await instanceSettingsService(db).updateExperimental({ enableManagedSandboxOnly: true });

    const run = await heartbeat.invoke(fixture.agentId, "assignment", { issueId: fixture.issueId }, "system");
    expect(run).not.toBeNull();
    await heartbeat.drainActiveRunExecutions();

    const finished = await heartbeat.getRun(run!.id);
    expect(finished?.status).toBe("failed");
    expect(finished?.error).toContain("no active managed sandbox environment exists");
    expect(executedRunIds).not.toContain(run!.id);
    expect(await db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, run!.id))).toHaveLength(0);
    expect((await heartbeat.getRun(fixture.holderRunId))?.status).toBe("running");
  });

  it("redirects a local selection to the managed sandbox before applying workspace contention", async () => {
    const fixture = await seedWorkspaceFixture({ agentEnvironmentDriver: "local" });
    await db.insert(environments).values({
      name: "Managed sandbox", driver: "sandbox", status: "active",
      config: { provider: "fake" }, metadata: { managedByPaperclip: true },
    });
    await instanceSettingsService(db).updateExperimental({ enableManagedSandboxOnly: true });

    const run = await heartbeat.invoke(fixture.agentId, "assignment", { issueId: fixture.issueId }, "system");
    expect(run).not.toBeNull();
    await heartbeat.drainActiveRunExecutions();

    expect((await heartbeat.getRun(run!.id))?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE);
    expect(executedRunIds).not.toContain(run!.id);
    expect(await db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, run!.id))).toHaveLength(0);
    expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run!.id))).toHaveLength(1);
    expect((await heartbeat.getRun(fixture.holderRunId))?.status).toBe("running");
  });

  it.each([
    { driver: "local", policySource: "issue" },
    { driver: "local", policySource: "project" },
    { driver: "ssh", policySource: "issue" },
    { driver: "ssh", policySource: "project" },
  ] as const)("dispatches on $driver despite a $policySource serialize policy", async ({ driver, policySource }) => {
    const fixture = await seedWorkspaceFixture({
      agentEnvironmentDriver: driver,
      issueWorkspaceSettings: policySource === "issue" ? { sharedWorkspaceConcurrency: "serialize" } : null,
      projectWorkspacePolicy: policySource === "project"
        ? { enabled: true, defaultMode: "shared_workspace", sharedWorkspaceConcurrency: "serialize" }
        : undefined,
    });

    const run = await heartbeat.invoke(
      fixture.agentId,
      "assignment",
      { issueId: fixture.issueId, wakeReason: "issue_assigned" },
      "system",
    );
    expect(run).not.toBeNull();

    const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
    expect(finishedRun?.status).toBe("succeeded");
    expect(finishedRun?.errorCode).not.toBe(WORKSPACE_BUSY_ERROR_CODE);
    expect(executedRunIds).toContain(run!.id);
    expect(executedInputs.get(run!.id)?.context.paperclipTaskMarkdown).toContain(
      `shared workspace is concurrently held by run ${fixture.holderRunId}`,
    );
    expect((await heartbeat.getRun(fixture.holderRunId))?.status).toBe("running");
    const retryRuns = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(
      and(eq(heartbeatRuns.companyId, fixture.companyId), eq(heartbeatRuns.scheduledRetryReason, WORKSPACE_BUSY_RETRY_REASON)),
    );
    expect(retryRuns).toHaveLength(0);
  });

  it("rejects unproven concurrent physical sharing in a sandbox environment", async () => {
    const fixture = await seedWorkspaceFixture({
      issueWorkspaceSettings: { sharedWorkspaceConcurrency: "allow" },
      agentEnvironmentDriver: "sandbox",
    });

    const run = await heartbeat.invoke(
      fixture.agentId,
      "assignment",
      { issueId: fixture.issueId, wakeReason: "issue_assigned" },
      "system",
    );
    expect(run).not.toBeNull();

    const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
    expect(executedRunIds).not.toContain(run!.id);
    expect(finishedRun?.error).toContain("cannot place concurrent agents in one physical shared folder");
    const retryRuns = await db
      .select({ id: heartbeatRuns.id })
      .from(heartbeatRuns)
      .where(eq(heartbeatRuns.scheduledRetryReason, WORKSPACE_BUSY_RETRY_REASON));
    expect(retryRuns).toHaveLength(0);
  });

  describe("reused shared execution workspace in a sandbox", () => {
    // The holder uses a different project workspace so only the shared execution
    // workspace connects the two runs.
    const seedInheritedWorkspace = (input?: Parameters<typeof seedWorkspaceFixture>[0]) => seedWorkspaceFixture({
      sharedExecutionWorkspace: true,
      holderProjectWorkspaceId: randomUUID(),
      issueWorkspaceSettings: { mode: "shared_workspace" },
      ...input,
    });

    it("defers instead of failing while a live run holds the workspace", async () => {
      const fixture = await seedInheritedWorkspace();

      const run = await heartbeat.invoke(
        fixture.agentId,
        "assignment",
        { issueId: fixture.issueId, wakeReason: "issue_assigned" },
        "system",
      );
      expect(run).not.toBeNull();

      const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
      expect(finishedRun?.status).toBe("cancelled");
      expect(finishedRun?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE);
      expect(finishedRun?.error).not.toContain("cannot place concurrent agents in one physical shared folder");
      expect(finishedRun?.resultJson).toMatchObject({
        executionRecovery: { kind: "workspace_wait", providerWorkStarted: false },
        cancellation: { reason: "Waiting for the shared execution workspace" },
        workspaceBusy: {
          projectWorkspaceId: null,
          executionWorkspaceId: fixture.sharedExecutionWorkspaceId,
          holderRunId: fixture.holderRunId,
          holderIssueId: fixture.holderIssueId,
          deferralAttempt: 0,
        },
      });
      expect(executedRunIds).not.toContain(run!.id);
      expect(await waitForRetryRun(run!.id)).toMatchObject({
        status: "scheduled_retry",
        scheduledRetryReason: WORKSPACE_BUSY_RETRY_REASON,
      });
      await waitForPendingRunFailureReports();
      expect(captureRunFailure.mock.calls.filter(([report]) => report.runId === run!.id)).toHaveLength(0);
      const [issueRow] = await db.select({ status: issues.status }).from(issues).where(eq(issues.id, fixture.issueId));
      expect(issueRow?.status).not.toBe("blocked");
    });

    it("resumes in the inherited workspace once the holder finishes", async () => {
      const fixture = await seedInheritedWorkspace();

      const run = await heartbeat.invoke(
        fixture.agentId,
        "assignment",
        { issueId: fixture.issueId, wakeReason: "issue_assigned" },
        "system",
      );
      expect(run).not.toBeNull();
      expect((await waitForRunToLeaveActiveStates(run!.id))?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE);
      const retryRun = await waitForRetryRun(run!.id);
      expect(retryRun?.status).toBe("scheduled_retry");

      await db
        .update(heartbeatRuns)
        .set({ status: "succeeded", finishedAt: new Date() })
        .where(eq(heartbeatRuns.id, fixture.holderRunId));
      const afterDue = new Date(new Date(retryRun!.scheduledRetryAt!).getTime() + 1_000);
      expect((await heartbeat.promoteDueScheduledRetries(afterDue)).runIds).toContain(retryRun!.id);
      await heartbeat.resumeQueuedRuns();

      expect((await waitForRunToLeaveActiveStates(retryRun!.id))?.status).toBe("succeeded");
      expect(executedRunIds).toContain(retryRun!.id);
      expect(executedInputs.get(retryRun!.id)?.context.paperclipWorkspace).toMatchObject({
        cwd: workspaceCwd,
        mode: "shared_workspace",
      });
      const [issueRow] = await db
        .select({ executionWorkspaceId: issues.executionWorkspaceId })
        .from(issues)
        .where(eq(issues.id, fixture.issueId));
      expect(issueRow?.executionWorkspaceId).toBe(fixture.sharedExecutionWorkspaceId);
    });

    it("still rejects an explicit allow policy that requires concurrent physical sharing", async () => {
      const fixture = await seedInheritedWorkspace({
        issueWorkspaceSettings: { mode: "shared_workspace", sharedWorkspaceConcurrency: "allow" },
      });

      const run = await heartbeat.invoke(
        fixture.agentId,
        "assignment",
        { issueId: fixture.issueId, wakeReason: "issue_assigned" },
        "system",
      );
      expect(run).not.toBeNull();

      const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
      expect(finishedRun?.status).toBe("failed");
      expect(finishedRun?.error).toContain("cannot place concurrent agents in one physical shared folder");
      expect(executedRunIds).not.toContain(run!.id);
      expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
        eq(heartbeatRuns.companyId, fixture.companyId),
        eq(heartbeatRuns.scheduledRetryReason, WORKSPACE_BUSY_RETRY_REASON),
      ))).toHaveLength(0);
    });

    it("neither overlaps nor waits forever on a quiet holder that is still running", async () => {
      const fixture = await seedInheritedWorkspace({
        holderActivityAt: new Date(Date.now() - WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS - 60_000),
      });

      const run = await heartbeat.invoke(
        fixture.agentId,
        "assignment",
        { issueId: fixture.issueId, wakeReason: "issue_assigned" },
        "system",
      );
      expect(run).not.toBeNull();

      const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
      expect(finishedRun?.status).toBe("failed");
      expect(finishedRun?.errorCode).not.toBe(WORKSPACE_BUSY_ERROR_CODE);
      expect(finishedRun?.error).toContain(`run ${fixture.holderRunId} still holds this shared execution workspace`);
      expect(finishedRun?.resultJson).toMatchObject({
        configurationIncomplete: { reason: "shared_realization_unsupported", holderRunId: fixture.holderRunId, holderStale: true },
      });
      expect(executedRunIds).not.toContain(run!.id);
      expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
        eq(heartbeatRuns.companyId, fixture.companyId),
        eq(heartbeatRuns.scheduledRetryReason, WORKSPACE_BUSY_RETRY_REASON),
      ))).toHaveLength(0);
      expect((await heartbeat.getRun(fixture.holderRunId))?.status).toBe("running");
    });
  });

  it.each([0, 35 * 60_000])("defers a run in a busy shared workspace after %i ms of holder silence", async (silenceMs) => {
    const fixture = await seedWorkspaceFixture({ holderActivityAt: new Date(Date.now() - silenceMs) });

    const run = await heartbeat.invoke(
      fixture.agentId,
      "assignment",
      { issueId: fixture.issueId, wakeReason: "issue_assigned" },
      "system",
    );
    expect(run).not.toBeNull();

    const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
    expect(finishedRun?.status).toBe("cancelled");
    expect(finishedRun?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE);
    const workspaceBusy = (finishedRun?.resultJson as Record<string, unknown> | null)
      ?.workspaceBusy as Record<string, unknown> | undefined;
    expect(workspaceBusy).toMatchObject({
      projectWorkspaceId: fixture.projectWorkspaceId,
      holderRunId: fixture.holderRunId,
      holderIssueId: fixture.holderIssueId,
      deferralAttempt: 0,
    });
    expect(finishedRun?.resultJson).toMatchObject({
      executionRecovery: { kind: "workspace_wait", providerWorkStarted: false },
      cancellation: {
        source: "control_plane",
        expected: true,
        initiator: { type: "system" },
        reason: "Waiting for the shared project workspace",
        recordedAt: finishedRun!.finishedAt!.toISOString(),
      },
    });

    // The deferred run's adapter never executed — the whole point of the gate.
    expect(executedRunIds).not.toContain(run!.id);

    const retryRun = await waitForRetryRun(run!.id);
    expect(retryRun).toMatchObject({
      status: "scheduled_retry",
      scheduledRetryAttempt: 1,
      scheduledRetryReason: WORKSPACE_BUSY_RETRY_REASON,
    });
    await waitForPendingRunFailureReports();
    expect(captureRunFailure.mock.calls.filter(([report]) => report.runId === run!.id)).toHaveLength(0);
    expect((retryRun?.contextSnapshot as Record<string, unknown> | null)?.wakeReason).toBe(
      WORKSPACE_BUSY_RETRY_WAKE_REASON,
    );

    // The issue execution lock moved to the retry run, so the issue keeps an
    // active execution path and stranded-issue recovery leaves it alone.
    const issueRow = await db
      .select({ executionRunId: issues.executionRunId })
      .from(issues)
      .where(eq(issues.id, fixture.issueId))
      .then((rows) => rows[0] ?? null);
    expect(issueRow?.executionRunId).toBe(retryRun!.id);

    const finishedAtMs = finishedRun?.finishedAt ? new Date(finishedRun.finishedAt).getTime() : 0;
    const dueAtMs = retryRun?.scheduledRetryAt ? new Date(retryRun.scheduledRetryAt).getTime() : 0;
    expect(dueAtMs).toBeGreaterThanOrEqual(finishedAtMs + WORKSPACE_BUSY_RETRY_BASE_DELAY_MS);
    expect(dueAtMs).toBeLessThanOrEqual(
      finishedAtMs + WORKSPACE_BUSY_RETRY_BASE_DELAY_MS + WORKSPACE_BUSY_RETRY_JITTER_MS,
    );

    // The holder was left undisturbed and the agent returned to idle rather
    // than error.
    const holderRun = await heartbeat.getRun(fixture.holderRunId);
    expect(holderRun?.status).toBe("running");
    await expect
      .poll(
        () =>
          db
            .select({ status: agents.status, errorReason: agents.errorReason })
            .from(agents)
            .where(eq(agents.id, fixture.agentId))
            .then((rows) => rows[0] ?? null),
        { timeout: 5_000, interval: 50 },
      )
      .toEqual({ status: "idle", errorReason: null });

    if (run!.wakeupRequestId) {
      const wakeup = await db
        .select({ status: agentWakeupRequests.status })
        .from(agentWakeupRequests)
        .where(eq(agentWakeupRequests.id, run!.wakeupRequestId))
        .then((rows) => rows[0] ?? null);
      expect(wakeup?.status).toBe("cancelled");
    }
  });

  it.each([false, true])("executes the scheduled retry without inheriting expected cancellation (adapter fails: %s)", async (adapterFails) => {
    const fixture = await seedWorkspaceFixture();

    const run = await heartbeat.invoke(
      fixture.agentId,
      "assignment",
      { issueId: fixture.issueId, wakeReason: "issue_assigned" },
      "system",
    );
    expect(run).not.toBeNull();
    const deferred = await waitForRunToLeaveActiveStates(run!.id);
    expect(deferred?.status).toBe("cancelled");

    const retryRun = await waitForRetryRun(run!.id);
    expect(retryRun?.status).toBe("scheduled_retry");
    expect(retryRun?.resultJson?.cancellation).toBeUndefined();
    if (adapterFails) failExecutionRunIds.add(retryRun!.id);

    // Holder finishes; the due retry promotes, queues, and executes.
    await db
      .update(heartbeatRuns)
      .set({ status: "succeeded", finishedAt: new Date() })
      .where(eq(heartbeatRuns.id, fixture.holderRunId));

    const afterDue = new Date(new Date(retryRun!.scheduledRetryAt!).getTime() + 1_000);
    const promotion = await heartbeat.promoteDueScheduledRetries(afterDue);
    expect(promotion.runIds).toContain(retryRun!.id);

    await heartbeat.resumeQueuedRuns();
    const finishedRetry = await waitForRunToLeaveActiveStates(retryRun!.id);
    expect(finishedRetry?.status).toBe(adapterFails ? "failed" : "succeeded");
    expect(executedRunIds).toContain(retryRun!.id);
    expect(finishedRetry?.resultJson?.cancellation).toBeUndefined();
    if (adapterFails) {
      await expect.poll(() => captureRunFailure.mock.calls.filter(([report]) => report.runId === retryRun!.id)).toHaveLength(1);
      expect(captureRunFailure).toHaveBeenCalledWith(expect.objectContaining({
        runId: retryRun!.id,
        runStatus: "failed",
        exitCode: 1,
      }));
    }
    await waitForPendingRunFailureReports();
    expect(captureRunFailure.mock.calls.filter(([report]) => report.runId === run!.id)).toHaveLength(0);
  });

  it("does not hide ambiguous, malformed, provider, or transport cancellations behind workspace_busy", async () => {
    const fixture = await seedWorkspaceFixture();
    const run = await heartbeat.invoke(fixture.agentId, "assignment", {
      issueId: fixture.issueId, wakeReason: "issue_assigned",
    }, "system");
    expect(run).not.toBeNull();
    await waitForRunToLeaveActiveStates(run!.id);
    await waitForRetryRun(run!.id);
    const [deferred] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id));
    expect(deferred?.status).toBe("cancelled");
    const receipt = deferred!.resultJson!.cancellation as Record<string, unknown>;
    for (const cancellation of [
      undefined,
      { ...receipt, expected: "true" },
      { ...receipt, expected: false, source: "provider", initiator: { type: "provider" } },
      { ...receipt, expected: false, source: "transport" },
    ]) {
      captureRunFailure.mockClear();
      const unexpected = {
        ...deferred!, id: randomUUID(),
        resultJson: { ...deferred!.resultJson, cancellation },
      };
      await reportRunFailure(db, unexpected);
      expect(captureRunFailure).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
        runId: unexpected.id,
        runStatus: "cancelled",
        errorCode: WORKSPACE_BUSY_ERROR_CODE,
      }));
    }
  });

  it("preserves a legacy accepted mention run through workspace retry", async () => {
    const fixture = await seedWorkspaceFixture();

    // Existing queued work can contain coalesced assignment or feedback.
    // Only new mention requests are ignored; keep accepted work's rules.
    const [run] = await db.insert(heartbeatRuns).values({
      companyId: fixture.companyId, agentId: fixture.nonAssigneeAgentId,
      invocationSource: "on_demand", status: "queued", responsibleUserId: "responsible-user",
      contextSnapshot: {
        issueId: fixture.issueId, wakeReason: "issue_comment_mentioned", commentId: randomUUID(),
      },
    }).returning();
    await heartbeat.resumeQueuedRuns();

    const deferred = await waitForRunToLeaveActiveStates(run!.id);
    expect(deferred?.status).toBe("cancelled");
    expect(deferred?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE);
    expect(executedRunIds).not.toContain(run!.id);

    const retryRun = await waitForRetryRun(run!.id);
    expect(retryRun).toMatchObject({
      status: "scheduled_retry",
      scheduledRetryReason: WORKSPACE_BUSY_RETRY_REASON,
    });
    expect(
      (retryRun?.contextSnapshot as Record<string, unknown> | null)?.workspaceBusyDeferredWhileAssignee,
    ).toBe(false);

    // The non-assignee run never held the issue execution lock, so the
    // deferral must not have stolen or released it.
    const issueRow = await db
      .select({ executionRunId: issues.executionRunId })
      .from(issues)
      .where(eq(issues.id, fixture.issueId))
      .then((rows) => rows[0] ?? null);
    expect(issueRow?.executionRunId).toBeNull();

    // The holder finishes; the retry must survive the promotion gate even
    // though the retry's agent is not the issue assignee.
    await db
      .update(heartbeatRuns)
      .set({ status: "succeeded", finishedAt: new Date() })
      .where(eq(heartbeatRuns.id, fixture.holderRunId));

    const afterDue = new Date(new Date(retryRun!.scheduledRetryAt!).getTime() + 1_000);
    const promotion = await heartbeat.promoteDueScheduledRetries(afterDue);
    expect(promotion.runIds).toContain(retryRun!.id);

    await heartbeat.resumeQueuedRuns();
    const finishedRetry = await waitForRunToLeaveActiveStates(retryRun!.id);
    expect(finishedRetry?.status).toBe("succeeded");
    expect(executedRunIds).toContain(retryRun!.id);
  });

  it("still cancels an assignee retry at promotion when the issue is reassigned", async () => {
    const fixture = await seedWorkspaceFixture();

    const run = await heartbeat.invoke(
      fixture.agentId,
      "assignment",
      { issueId: fixture.issueId, wakeReason: "issue_assigned" },
      "system",
    );
    expect(run).not.toBeNull();
    const deferred = await waitForRunToLeaveActiveStates(run!.id);
    expect(deferred?.status).toBe("cancelled");

    const retryRun = await waitForRetryRun(run!.id);
    expect(retryRun?.status).toBe("scheduled_retry");
    expect(
      (retryRun?.contextSnapshot as Record<string, unknown> | null)?.workspaceBusyDeferredWhileAssignee,
    ).toBe(true);

    // Ownership changes while the retry pends: the deferral was taken under
    // assignee-ship, so the reassignment protection must still cancel it.
    await db
      .update(issues)
      .set({ assigneeAgentId: fixture.nonAssigneeAgentId, executionRunId: null })
      .where(eq(issues.id, fixture.issueId));

    const afterDue = new Date(new Date(retryRun!.scheduledRetryAt!).getTime() + 1_000);
    const promotion = await heartbeat.promoteDueScheduledRetries(afterDue);
    expect(promotion.runIds).not.toContain(retryRun!.id);

    const cancelledRetry = await heartbeat.getRun(retryRun!.id);
    expect(cancelledRetry?.status).toBe("cancelled");
    expect(cancelledRetry?.errorCode).toBe("issue_reassigned");
  });

  it("does not defer when the holder issue explicitly uses an isolated workspace", async () => {
    await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true });
    try {
      const fixture = await seedWorkspaceFixture({
        holderIssueWorkspaceSettings: { mode: "isolated_workspace" },
      });

      const run = await heartbeat.invoke(
        fixture.agentId,
        "assignment",
        { issueId: fixture.issueId, wakeReason: "issue_assigned" },
        "system",
      );
      expect(run).not.toBeNull();

      const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
      expect(finishedRun?.errorCode).not.toBe(WORKSPACE_BUSY_ERROR_CODE);
      const retryRuns = await db
        .select({ id: heartbeatRuns.id })
        .from(heartbeatRuns)
        .where(
          and(
            eq(heartbeatRuns.companyId, fixture.companyId),
            eq(heartbeatRuns.scheduledRetryReason, WORKSPACE_BUSY_RETRY_REASON),
          ),
        )
        .then((rows) => rows.length);
      expect(retryRuns).toBe(0);
    } finally {
      await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: false });
    }
  });

  it("does not defer when the running run holds a different project workspace", async () => {
    const fixture = await seedWorkspaceFixture({ holderProjectWorkspaceId: randomUUID() });

    const run = await heartbeat.invoke(
      fixture.agentId,
      "assignment",
      { issueId: fixture.issueId, wakeReason: "issue_assigned" },
      "system",
    );
    expect(run).not.toBeNull();

    const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
    expect(finishedRun?.errorCode).not.toBe(WORKSPACE_BUSY_ERROR_CODE);
    const retryRuns = await db
      .select({ id: heartbeatRuns.id })
      .from(heartbeatRuns)
      .where(
        and(
          eq(heartbeatRuns.companyId, fixture.companyId),
          eq(heartbeatRuns.scheduledRetryReason, WORKSPACE_BUSY_RETRY_REASON),
        ),
      )
      .then((rows) => rows.length);
    expect(retryRuns).toBe(0);
  });

  it("does not defer when the only holder has been silent past the staleness threshold", async () => {
    const fixture = await seedWorkspaceFixture({
      holderActivityAt: new Date(Date.now() - WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS - 60_000),
    });

    const run = await heartbeat.invoke(
      fixture.agentId,
      "assignment",
      { issueId: fixture.issueId, wakeReason: "issue_assigned" },
      "system",
    );
    expect(run).not.toBeNull();

    // The holder row is still "running", but it has been silent past the
    // staleness bar, so it no longer blocks the workspace.
    const finishedRun = await waitForRunToLeaveActiveStates(run!.id);
    expect(finishedRun?.status).toBe("succeeded");
    expect(finishedRun?.errorCode).not.toBe(WORKSPACE_BUSY_ERROR_CODE);
    expect(executedRunIds).toContain(run!.id);
  });

  it("keeps deferring past earlier attempts while the holder is still live", async () => {
    const fixture = await seedWorkspaceFixture();

    // Seed the promoted continuation of a run that has already been deferred
    // many times while the holder is still live: it must defer again, not
    // dispatch alongside the live holder.
    const priorAttempts = 10;
    const priorRunId = randomUUID();
    const wakeupId = randomUUID();
    const retryRunId = randomUUID();
    const now = new Date();
    const dueAt = new Date(now.getTime() - 1_000);

    await db.insert(heartbeatRuns).values({
      id: priorRunId,
      companyId: fixture.companyId,
      agentId: fixture.agentId,
      invocationSource: "assignment",
      triggerDetail: "system",
      status: "cancelled",
      errorCode: WORKSPACE_BUSY_ERROR_CODE,
      finishedAt: now,
      scheduledRetryAttempt: priorAttempts - 1,
      scheduledRetryReason: WORKSPACE_BUSY_RETRY_REASON,
      contextSnapshot: {
        issueId: fixture.issueId,
        wakeReason: WORKSPACE_BUSY_RETRY_WAKE_REASON,
      },
      createdAt: now,
      updatedAt: now,
    });

    await db.insert(agentWakeupRequests).values({
      id: wakeupId,
      companyId: fixture.companyId,
      agentId: fixture.agentId,
      source: "automation",
      triggerDetail: "system",
      reason: WORKSPACE_BUSY_RETRY_WAKE_REASON,
      payload: {
        issueId: fixture.issueId,
        retryOfRunId: priorRunId,
        retryReason: WORKSPACE_BUSY_RETRY_REASON,
        scheduledRetryAttempt: priorAttempts,
      },
      status: "queued",
      requestedByActorType: "system",
    });

    await db.insert(heartbeatRuns).values({
      id: retryRunId,
      companyId: fixture.companyId,
      agentId: fixture.agentId,
      invocationSource: "automation",
      triggerDetail: "system",
      status: "scheduled_retry",
      wakeupRequestId: wakeupId,
      retryOfRunId: priorRunId,
      scheduledRetryAt: dueAt,
      scheduledRetryAttempt: priorAttempts,
      scheduledRetryReason: WORKSPACE_BUSY_RETRY_REASON,
      contextSnapshot: {
        issueId: fixture.issueId,
        wakeReason: WORKSPACE_BUSY_RETRY_WAKE_REASON,
        retryReason: WORKSPACE_BUSY_RETRY_REASON,
        workspaceBusyDeferredWhileAssignee: true,
      },
      createdAt: now,
      updatedAt: now,
    });
    await db
      .update(agentWakeupRequests)
      .set({ runId: retryRunId })
      .where(eq(agentWakeupRequests.id, wakeupId));

    const promotion = await heartbeat.promoteDueScheduledRetries(now);
    expect(promotion.runIds).toContain(retryRunId);

    await heartbeat.resumeQueuedRuns();
    const finishedRun = await waitForRunToLeaveActiveStates(retryRunId);

    expect(finishedRun?.status).toBe("cancelled");
    expect(finishedRun?.errorCode).toBe(WORKSPACE_BUSY_ERROR_CODE);
    expect(executedRunIds).not.toContain(retryRunId);

    const nextRetry = await waitForRetryRun(retryRunId);
    expect(nextRetry).toMatchObject({
      status: "scheduled_retry",
      scheduledRetryAttempt: priorAttempts + 1,
      scheduledRetryReason: WORKSPACE_BUSY_RETRY_REASON,
    });

    const holderRun = await heartbeat.getRun(fixture.holderRunId);
    expect(holderRun?.status).toBe("running");
  });
});


enableRunFailureReporting();