PaperClipAI/server/src/__tests__/workspace-operations-reconciliation.test.ts

import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
  companies,
  createDb,
  executionWorkspaces,
  projects,
  workspaceOperations,
} from "@paperclipai/db";
import { eq } from "drizzle-orm";
import { getWorkspaceOperationLogStore } from "../services/workspace-operation-log-store.js";
import {
  RUNTIME_CONTROL_STALE_AFTER_MS,
  resetWorkspaceRuntimeControlLocksForTests,
  runExclusiveWorkspaceRuntimeControl,
  workspaceOperationService,
} from "../services/workspace-operations.js";
import {
  getEmbeddedPostgresTestSupport,
  startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";

describe("workspace runtime control serialization", () => {
  afterEach(() => {
    resetWorkspaceRuntimeControlLocksForTests();
  });

  it("deterministically rejects an overlapping control operation and releases the claim", async () => {
    let releaseFirst!: () => void;
    const firstGate = new Promise<void>((resolve) => {
      releaseFirst = resolve;
    });
    const first = runExclusiveWorkspaceRuntimeControl({
      executionWorkspaceId: "workspace-1",
      action: "start",
      run: async () => {
        await firstGate;
        return "started";
      },
    });

    await expect(runExclusiveWorkspaceRuntimeControl({
      executionWorkspaceId: "workspace-1",
      action: "repair",
      run: async () => "repaired",
    })).rejects.toMatchObject({
      status: 409,
      details: {
        code: "workspace_runtime_control_in_progress",
        activeAction: "start",
        requestedAction: "repair",
      },
    });

    releaseFirst();
    await expect(first).resolves.toBe("started");
    await expect(runExclusiveWorkspaceRuntimeControl({
      executionWorkspaceId: "workspace-1",
      action: "stop",
      run: async () => "stopped",
    })).resolves.toBe("stopped");
  });
});

const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_WORKSPACE_OPERATIONS_TEST_DATABASE_URL");
const embeddedPostgresSupport = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;

if (!embeddedPostgresSupport.supported) {
  console.warn(
    `Skipping workspace-operation reconciliation tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
  );
}

describeEmbeddedPostgres("workspace operation reconciliation", () => {
  let db!: ReturnType<typeof createDb>;
  let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
  let logRoot = "";
  let previousLogRoot: string | undefined;

  beforeAll(async () => {
    if (!externalUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-workspace-operation-reconcile-");
    db = createDb(externalUrl ?? tempDb!.connectionString);
    logRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-operation-logs-"));
    previousLogRoot = process.env.WORKSPACE_OPERATION_LOG_BASE_PATH;
    process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = logRoot;
  }, 20_000);

  afterEach(async () => {
    await db.delete(workspaceOperations);
    await db.delete(executionWorkspaces);
    await db.delete(projects);
    await db.delete(companies);
  });

  afterAll(async () => {
    if (previousLogRoot === undefined) delete process.env.WORKSPACE_OPERATION_LOG_BASE_PATH;
    else process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = previousLogRoot;
    await fs.rm(logRoot, { recursive: true, force: true });
    await db?.$client.end({ timeout: 1 });
    await tempDb?.cleanup();
  });

  it("terminalizes an orphaned runtime start with an inspectable reconciliation log", async () => {
    const companyId = randomUUID();
    const projectId = randomUUID();
    const executionWorkspaceId = randomUUID();
    const staleOperationId = randomUUID();
    const unrelatedOperationId = randomUUID();

    await db.insert(companies).values({
      id: companyId,
      name: "Workspace operation reconciliation",
      issuePrefix: `W${companyId.replace(/-/g, "").slice(0, 7).toUpperCase()}`,
      requireBoardApprovalForNewAgents: false,
    });
    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: "Runtime reconciliation",
      status: "in_progress",
    });
    await db.insert(executionWorkspaces).values({
      id: executionWorkspaceId,
      companyId,
      projectId,
      mode: "isolated_workspace",
      strategyType: "git_worktree",
      name: "Reconciliation workspace",
      status: "active",
      cwd: "/tmp/reconciliation-workspace",
    });

    const logHandle = await getWorkspaceOperationLogStore().begin({
      companyId,
      operationId: staleOperationId,
    });
    await db.insert(workspaceOperations).values([
      {
        id: staleOperationId,
        companyId,
        executionWorkspaceId,
        phase: "workspace_provision",
        command: "workspace command start",
        cwd: "/tmp/reconciliation-workspace",
        status: "running",
        logStore: logHandle.store,
        logRef: logHandle.logRef,
        metadata: { action: "start", executionWorkspaceId },
      },
      {
        id: unrelatedOperationId,
        companyId,
        executionWorkspaceId,
        phase: "workspace_provision",
        command: "bash ./scripts/provision-worktree.sh",
        cwd: "/tmp/reconciliation-workspace",
        status: "running",
        metadata: { created: true },
      },
    ]);

    const service = workspaceOperationService(db);

    // Recovery is bounded and cannot steal a live operation (PAP-17249): a `running` control
    // that has signalled recently is left alone even when it carries no ownership stamp, which
    // is the shape a control recorded by a pre-stamp build has.
    await expect(service.reconcileStaleRuntimeControlOperations(executionWorkspaceId)).resolves.toEqual({
      reconciled: 0,
      operationIds: [],
    });
    expect((await service.getById(staleOperationId))?.status).toBe("running");

    // Once it has gone quiet for longer than the staleness window, it is terminalized.
    await expect(
      service.reconcileStaleRuntimeControlOperations(executionWorkspaceId, {
        now: new Date(Date.now() + RUNTIME_CONTROL_STALE_AFTER_MS + 5_000),
      }),
    ).resolves.toEqual({
      reconciled: 1,
      operationIds: [staleOperationId],
    });

    const stale = await service.getById(staleOperationId);
    expect(stale).toMatchObject({
      status: "failed",
      metadata: {
        action: "start",
        reconciled: true,
        reconciliationReason: "orphaned_runtime_control",
      },
    });
    expect(stale?.finishedAt).toBeInstanceOf(Date);
    expect(stale?.stderrExcerpt).toContain("no live owner has reported progress");
    expect((await service.readLog(staleOperationId)).content).toContain(
      "no live owner has reported progress",
    );

    const unrelated = await db
      .select({ status: workspaceOperations.status })
      .from(workspaceOperations)
      .where(eq(workspaceOperations.id, unrelatedOperationId))
      .then((rows) => rows[0]);
    expect(unrelated?.status).toBe("running");
  });

  it("persists bounded repair phase progress on the operation row and log", async () => {
    const companyId = randomUUID();
    const projectId = randomUUID();
    const executionWorkspaceId = randomUUID();
    await db.insert(companies).values({
      id: companyId,
      name: "Workspace repair diagnostics",
      issuePrefix: `R${companyId.replace(/-/g, "").slice(0, 7).toUpperCase()}`,
      requireBoardApprovalForNewAgents: false,
    });
    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: "Repair diagnostics",
      status: "in_progress",
    });
    await db.insert(executionWorkspaces).values({
      id: executionWorkspaceId,
      companyId,
      projectId,
      mode: "isolated_workspace",
      strategyType: "git_worktree",
      name: "Repair diagnostics workspace",
      status: "active",
      cwd: "/tmp/repair-diagnostics-workspace",
    });

    const operation = await workspaceOperationService(db)
      .createRecorder({ companyId, executionWorkspaceId })
      .recordOperation({
        phase: "workspace_repair",
        command: "workspace command repair",
        cwd: "/tmp/repair-diagnostics-workspace",
        metadata: { action: "repair" },
        run: async (reportProgress) => {
          await reportProgress({
            metadata: {
              repairPhase: "target_backup",
              repairDiagnostics: [{
                phase: "target_backup",
                status: "succeeded",
                at: "2026-08-18T00:00:00.000Z",
              }],
              databaseOnly: true,
              worktreePreserved: true,
            },
            system: "Workspace repair target_backup: succeeded.\n",
          });
          return { status: "succeeded", metadata: { backupRetained: true } };
        },
      });

    expect(operation).toMatchObject({
      status: "succeeded",
      metadata: {
        action: "repair",
        repairPhase: "target_backup",
        databaseOnly: true,
        worktreePreserved: true,
        backupRetained: true,
      },
    });
    expect((await workspaceOperationService(db).readLog(operation.id)).content).toContain(
      "Workspace repair target_backup: succeeded.",
    );
  });

  describe("when the operation log file cannot be written", () => {
    // The log file is created, then every append hits a full disk (PAPERCLIP-BACKEND-4H).
    function failLogAppends() {
      const enospc = Object.assign(new Error("ENOSPC: no space left on device, write"), {
        code: "ENOSPC",
        errno: -28,
        syscall: "write",
      });
      const appendFile = fs.appendFile;
      const spy = vi.spyOn(fs, "appendFile").mockImplementation(async (file, ...rest) => {
        if (typeof file === "string" && file.startsWith(logRoot)) throw enospc;
        return appendFile(file, ...rest);
      });
      return { enospc, spy };
    }

    async function seedCompany() {
      const companyId = randomUUID();
      await db.insert(companies).values({
        id: companyId,
        name: "Workspace operation full disk",
        issuePrefix: `F${companyId.replace(/-/g, "").slice(0, 7).toUpperCase()}`,
        requireBoardApprovalForNewAgents: false,
      });
      return companyId;
    }

    async function operationRows(companyId: string) {
      return db.select().from(workspaceOperations).where(eq(workspaceOperations.companyId, companyId));
    }

    it("terminalizes the operation and surfaces the write error", async () => {
      const companyId = await seedCompany();
      const { enospc, spy } = failLogAppends();
      try {
        await expect(
          workspaceOperationService(db)
            .createRecorder({ companyId })
            .recordOperation({
              phase: "worktree_prepare",
              command: "git worktree add",
              cwd: "/tmp/full-disk-workspace",
              run: async () => ({ status: "succeeded", exitCode: 0, stdout: "Preparing worktree\n" }),
            }),
        ).rejects.toBe(enospc);
      } finally {
        spy.mockRestore();
      }

      const rows = await operationRows(companyId);
      expect(rows).toHaveLength(1);
      expect(rows[0]).toMatchObject({ status: "failed", phase: "worktree_prepare" });
      expect(rows[0]?.finishedAt).toBeInstanceOf(Date);
      expect(rows[0]?.stderrExcerpt).toContain("ENOSPC: no space left on device, write");
    });

    it("keeps the operation's own error instead of the log write error", async () => {
      const companyId = await seedCompany();
      const { spy } = failLogAppends();
      const gitError = new Error("fatal: could not create work tree dir");
      try {
        await expect(
          workspaceOperationService(db)
            .createRecorder({ companyId })
            .recordOperation({
              phase: "worktree_prepare",
              command: "git worktree add",
              cwd: "/tmp/full-disk-workspace",
              run: async () => {
                throw gitError;
              },
            }),
        ).rejects.toBe(gitError);
      } finally {
        spy.mockRestore();
      }

      const rows = await operationRows(companyId);
      expect(rows).toHaveLength(1);
      expect(rows[0]).toMatchObject({ status: "failed" });
      expect(rows[0]?.finishedAt).toBeInstanceOf(Date);
      expect(rows[0]?.stderrExcerpt).toContain("fatal: could not create work tree dir");
    });
  });
});