diff --git a/server/src/__tests__/agent-directory-lock-crash-point.test.ts b/server/src/__tests__/agent-directory-lock-crash-point.test.ts new file mode 100644 index 00000000000..85f87a86fb8 --- /dev/null +++ b/server/src/__tests__/agent-directory-lock-crash-point.test.ts @@ -0,0 +1,31 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +describe("directory-lock crash point", () => { + let root: string | null = null; + + afterEach(async () => { + vi.unstubAllEnvs(); + vi.resetModules(); + if (root) await rm(root, { recursive: true, force: true }); + root = null; + }); + + it("fails with the lock's own busy error, so preparation defers it like real contention", async () => { + root = await mkdtemp(path.join(os.tmpdir(), "directory-lock-crash-point-")); + const control = path.join(root, "crash-point.control"); + await writeFile(control, "t1 agent_directory.operation_lock.before_acquire:1 fail\n"); + vi.stubEnv("PAPERCLIP_CRASH_POINT_CONTROL", control); + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + vi.resetModules(); + const { agentDirectoryWorkingCopyService, AgentDirectoryOperationBusyError } = await import("../services/agent-directory-working-copies.js"); + const { prepareWithAgentDirectoryDeferral, AgentDirectoryPreparationDeferral } = await import("../services/heartbeat/agent-directory-busy.js"); + const service = agentDirectoryWorkingCopyService({} as never, async () => null, async row => row); + const row = { companyId: "company-1", runId: "run-1", agentId: "agent-1", location: "remote:sandbox-1", receipt: {} }; + const failure = service.collectStopped(row as never); + await expect(failure).rejects.toBeInstanceOf(AgentDirectoryOperationBusyError); + await expect(prepareWithAgentDirectoryDeferral(() => failure)).rejects.toBeInstanceOf(AgentDirectoryPreparationDeferral); + }); +}); diff --git a/server/src/services/agent-directory-working-copies.ts b/server/src/services/agent-directory-working-copies.ts index 8109e1ef45b..98ec9ab4ec8 100644 --- a/server/src/services/agent-directory-working-copies.ts +++ b/server/src/services/agent-directory-working-copies.ts @@ -659,7 +659,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string // Different controllers have different cache roots. Serialize their active // collection/cleanup operations in Postgres, without holding row locks over I/O. // Test instrumentation only (disarmed in production): restart chaos fails the Nth attempt with the lock's own conflict. - crashPoint("agent_directory.operation_lock.before_acquire", { fail: () => conflict("Agent directory operation is already active") }); + crashPoint("agent_directory.operation_lock.before_acquire", { fail: () => new AgentDirectoryOperationBusyError() }); return withDedicatedDbConnection(db, dedicated => dedicated.transaction(async tx => { const [lock] = await tx.execute(sql`select pg_try_advisory_xact_lock(hashtextextended(${`agent-directory:${row.companyId}:${row.agentId}`}, 0)) as acquired, (select setting::bigint from pg_settings where name = 'idle_in_transaction_session_timeout') as idle_timeout_ms`);